2023-10-04 08:10:35,661 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 2023-10-04 08:10:35,675 main DEBUG Took 0.011869 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-04 08:10:35,676 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-04 08:10:35,676 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-04 08:10:35,677 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-04 08:10:35,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,687 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-04 08:10:35,698 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,700 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,701 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,702 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,702 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,703 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,703 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,704 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,705 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,706 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,706 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,707 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,708 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:10:35,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,709 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-04 08:10:35,711 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:10:35,714 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-04 08:10:35,716 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-04 08:10:35,716 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-04 08:10:35,718 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-04 08:10:35,718 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-04 08:10:35,732 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-04 08:10:35,735 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-04 08:10:35,736 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-04 08:10:35,736 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-04 08:10:35,739 main DEBUG createAppenders(={Console}) 2023-10-04 08:10:35,740 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 initialized 2023-10-04 08:10:35,740 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 2023-10-04 08:10:35,741 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24269709 OK. 2023-10-04 08:10:35,741 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-04 08:10:35,742 main DEBUG OutputStream closed 2023-10-04 08:10:35,742 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-04 08:10:35,742 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-04 08:10:35,742 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@1fa268de OK 2023-10-04 08:10:35,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-04 08:10:35,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-04 08:10:35,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-04 08:10:35,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-04 08:10:35,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-04 08:10:35,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-04 08:10:35,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-04 08:10:35,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-04 08:10:35,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-04 08:10:35,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-04 08:10:35,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-04 08:10:35,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-04 08:10:35,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-04 08:10:35,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-04 08:10:35,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-04 08:10:35,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-04 08:10:35,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-04 08:10:35,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-04 08:10:35,797 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-04 08:10:35,797 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7205765b) with optional ClassLoader: null 2023-10-04 08:10:35,797 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-04 08:10:35,798 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@7205765b] started OK. 2023-10-04T08:10:35,999 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2023-10-04 08:10:36,003 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-04 08:10:36,003 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-04T08:10:36,091 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0 2023-10-04T08:10:36,120 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-04T08:10:36,125 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181, deleteOnExit=true 2023-10-04T08:10:36,125 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-04T08:10:36,126 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/test.cache.data in system properties and HBase conf 2023-10-04T08:10:36,127 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/hadoop.tmp.dir in system properties and HBase conf 2023-10-04T08:10:36,127 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/hadoop.log.dir in system properties and HBase conf 2023-10-04T08:10:36,128 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-04T08:10:36,129 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-04T08:10:36,129 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-04T08:10:36,227 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-04T08:10:36,593 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:10:36,597 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-04T08:10:36,597 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-04T08:10:36,597 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-04T08:10:36,598 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-04T08:10:36,599 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-04T08:10:36,599 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-04T08:10:36,600 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-04T08:10:36,600 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-04T08:10:36,601 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-04T08:10:36,601 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/nfs.dump.dir in system properties and HBase conf 2023-10-04T08:10:36,601 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/java.io.tmpdir in system properties and HBase conf 2023-10-04T08:10:36,602 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-04T08:10:36,602 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-04T08:10:36,603 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-04T08:10:37,066 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-04T08:10:37,070 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-04T08:10:37,354 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-04T08:10:37,491 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@7db00b5a via org.mortbay.log.Slf4jLog 2023-10-04T08:10:37,513 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-04T08:10:37,565 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-04T08:10:37,600 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/java.io.tmpdir/Jetty_localhost_39229_hdfs____hs1kzh/webapp 2023-10-04T08:10:37,740 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39229 2023-10-04T08:10:37,767 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-04T08:10:37,768 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-04T08:10:38,334 WARN [Listener at localhost/45837 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-04T08:10:38,393 WARN [Listener at localhost/45837 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-04T08:10:38,410 WARN [Listener at localhost/45837 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-04T08:10:38,415 INFO [Listener at localhost/45837 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-04T08:10:38,419 INFO [Listener at localhost/45837 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/java.io.tmpdir/Jetty_localhost_44837_datanode____.macu0t/webapp 2023-10-04T08:10:38,522 INFO [Listener at localhost/45837 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44837 2023-10-04T08:10:38,729 WARN [Listener at localhost/43533 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-04T08:10:38,999 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-04T08:10:39,158 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x93f09158aa7a465a: Processing first storage report for DS-3852f686-69f4-4819-98e9-7894b62bf37a from datanode 588ecdd4-8ff0-4951-ab0b-b49dcb70b795 2023-10-04T08:10:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x93f09158aa7a465a: from storage DS-3852f686-69f4-4819-98e9-7894b62bf37a node DatanodeRegistration(127.0.0.1:35837, datanodeUuid=588ecdd4-8ff0-4951-ab0b-b49dcb70b795, infoPort=35101, infoSecurePort=0, ipcPort=43533, storageInfo=lv=-57;cid=testClusterID;nsid=877844423;c=1696407037142), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-04T08:10:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x93f09158aa7a465a: Processing first storage report for DS-9094fa1e-9570-476f-8e49-d647b1689feb from datanode 588ecdd4-8ff0-4951-ab0b-b49dcb70b795 2023-10-04T08:10:39,160 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x93f09158aa7a465a: from storage DS-9094fa1e-9570-476f-8e49-d647b1689feb node DatanodeRegistration(127.0.0.1:35837, datanodeUuid=588ecdd4-8ff0-4951-ab0b-b49dcb70b795, infoPort=35101, infoSecurePort=0, ipcPort=43533, storageInfo=lv=-57;cid=testClusterID;nsid=877844423;c=1696407037142), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-04T08:10:39,165 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0 2023-10-04T08:10:39,234 INFO [Listener at localhost/43533 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181/zookeeper_0, clientPort=50391, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-04T08:10:39,249 INFO [Listener at localhost/43533 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50391 2023-10-04T08:10:39,259 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:39,261 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:39,941 INFO [Listener at localhost/43533 {}] util.FSUtils(462): Created version file at hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 with version=8 2023-10-04T08:10:39,941 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/hbase-staging 2023-10-04T08:10:40,132 DEBUG [Listener at localhost/43533 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-04T08:10:40,748 INFO [Listener at localhost/43533 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=9 2023-10-04T08:10:40,782 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:40,782 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:40,783 INFO [Listener at localhost/43533 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-04T08:10:40,783 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:40,783 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-04T08:10:40,931 INFO [Listener at localhost/43533 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-04T08:10:41,034 INFO [Listener at localhost/43533 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-04T08:10:41,051 DEBUG [Listener at localhost/43533 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-04T08:10:41,057 INFO [Listener at localhost/43533 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-04T08:10:41,107 DEBUG [Listener at localhost/43533 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 15994 (auto-detected) 2023-10-04T08:10:41,109 DEBUG [Listener at localhost/43533 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-04T08:10:41,147 INFO [Listener at localhost/43533 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:46181 2023-10-04T08:10:41,159 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:41,161 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:41,179 INFO [Listener at localhost/43533 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:46181 connecting to ZooKeeper ensemble=127.0.0.1:50391 2023-10-04T08:10:41,220 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:461810x0, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-04T08:10:41,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:46181-0x1017c868d4b0000 connected 2023-10-04T08:10:41,247 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-10-04T08:10:41,250 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-10-04T08:10:41,253 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-10-04T08:10:41,262 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46181 2023-10-04T08:10:41,262 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46181 2023-10-04T08:10:41,262 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46181 2023-10-04T08:10:41,263 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46181 2023-10-04T08:10:41,263 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46181 2023-10-04T08:10:41,273 INFO [Listener at localhost/43533 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62, hbase.cluster.distributed=false 2023-10-04T08:10:41,330 INFO [Listener at localhost/43533 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=9 2023-10-04T08:10:41,331 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:41,331 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:41,331 INFO [Listener at localhost/43533 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-04T08:10:41,331 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-04T08:10:41,331 INFO [Listener at localhost/43533 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-04T08:10:41,336 INFO [Listener at localhost/43533 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-04T08:10:41,338 INFO [Listener at localhost/43533 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-04T08:10:41,339 INFO [Listener at localhost/43533 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:45997 2023-10-04T08:10:41,341 INFO [Listener at localhost/43533 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-04T08:10:41,348 DEBUG [Listener at localhost/43533 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-04T08:10:41,349 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:41,351 INFO [Listener at localhost/43533 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:41,354 INFO [Listener at localhost/43533 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:45997 connecting to ZooKeeper ensemble=127.0.0.1:50391 2023-10-04T08:10:41,358 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:459970x0, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-04T08:10:41,359 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:45997-0x1017c868d4b0001 connected 2023-10-04T08:10:41,359 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-10-04T08:10:41,360 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-10-04T08:10:41,361 DEBUG [Listener at localhost/43533 {}] zookeeper.ZKUtil(113): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2023-10-04T08:10:41,362 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45997 2023-10-04T08:10:41,362 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45997 2023-10-04T08:10:41,363 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45997 2023-10-04T08:10:41,366 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45997 2023-10-04T08:10:41,366 DEBUG [Listener at localhost/43533 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45997 2023-10-04T08:10:41,368 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:41,377 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-10-04T08:10:41,377 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-10-04T08:10:41,378 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:41,397 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-10-04T08:10:41,397 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2023-10-04T08:10:41,397 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:41,397 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:41,398 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-10-04T08:10:41,400 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:46181 2023-10-04T08:10:41,400 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase3.apache.org,46181,1696407040117 from backup master directory 2023-10-04T08:10:41,400 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2023-10-04T08:10:41,404 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:41,404 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-10-04T08:10:41,404 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2023-10-04T08:10:41,405 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-04T08:10:41,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:41,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-04T08:10:41,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-04T08:10:41,486 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/hbase.id with ID: f42718e5-673d-4ffa-9f84-7438e0fb4f66 2023-10-04T08:10:41,529 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-04T08:10:41,546 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:41,546 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:41,588 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:10:41,590 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-04T08:10:41,606 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-04T08:10:41,607 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-04T08:10:41,608 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-04T08:10:41,616 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-04T08:10:41,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-04T08:10:41,681 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store 2023-10-04T08:10:42,103 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-04T08:10:42,104 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:10:42,104 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-04T08:10:42,104 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:10:42,104 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:10:42,105 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-04T08:10:42,105 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:10:42,105 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:10:42,105 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-04T08:10:42,107 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/.initializing 2023-10-04T08:10:42,107 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/WALs/jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:42,116 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-04T08:10:42,129 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46181%2C1696407040117, suffix=, logDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/WALs/jenkins-hbase3.apache.org,46181,1696407040117, archiveDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/oldWALs, maxLogs=10 2023-10-04T08:10:42,155 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/WALs/jenkins-hbase3.apache.org,46181,1696407040117/jenkins-hbase3.apache.org%2C46181%2C1696407040117.1696407042136, exclude list is [], retry=0 2023-10-04T08:10:42,181 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK] 2023-10-04T08:10:42,188 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-04T08:10:42,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/WALs/jenkins-hbase3.apache.org,46181,1696407040117/jenkins-hbase3.apache.org%2C46181%2C1696407040117.1696407042136 2023-10-04T08:10:42,253 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK]] 2023-10-04T08:10:42,253 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:10:42,254 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:10:42,257 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,314 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-04T08:10:42,350 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:10:42,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,356 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-04T08:10:42,357 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:10:42,358 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-04T08:10:42,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:10:42,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,364 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-04T08:10:42,364 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:10:42,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,371 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,386 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-04T08:10:42,390 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-04T08:10:42,394 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:10:42,395 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63062054, jitterRate=-0.060302168130874634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-04T08:10:42,402 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-04T08:10:42,403 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-04T08:10:42,443 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614b672a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:10:42,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-04T08:10:42,508 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-04T08:10:42,508 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-04T08:10:42,511 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-04T08:10:42,513 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-04T08:10:42,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 5 msec 2023-10-04T08:10:42,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-04T08:10:42,550 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-04T08:10:42,566 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2023-10-04T08:10:42,568 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2023-10-04T08:10:42,572 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-04T08:10:42,573 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2023-10-04T08:10:42,575 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2023-10-04T08:10:42,577 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-04T08:10:42,582 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2023-10-04T08:10:42,586 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2023-10-04T08:10:42,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2023-10-04T08:10:42,589 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2023-10-04T08:10:42,601 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-04T08:10:42,602 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2023-10-04T08:10:42,607 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-10-04T08:10:42,607 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2023-10-04T08:10:42,607 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,607 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,608 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,46181,1696407040117, sessionid=0x1017c868d4b0000, setting cluster-up flag (Was=false) 2023-10-04T08:10:42,623 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,623 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,629 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2023-10-04T08:10:42,630 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:42,635 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,635 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:10:42,642 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2023-10-04T08:10:42,643 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:10:42,684 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:45997 2023-10-04T08:10:42,686 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(997): ClusterId : f42718e5-673d-4ffa-9f84-7438e0fb4f66 2023-10-04T08:10:42,690 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-04T08:10:42,697 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-04T08:10:42,697 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-04T08:10:42,702 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-04T08:10:42,703 DEBUG [RS:0;jenkins-hbase3:45997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37fcd346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:10:42,705 DEBUG [RS:0;jenkins-hbase3:45997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2397a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-04T08:10:42,711 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-04T08:10:42,711 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-04T08:10:42,711 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-04T08:10:42,715 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,46181,1696407040117 with isa=jenkins-hbase3.apache.org/172.31.12.81:45997, startcode=1696407041329 2023-10-04T08:10:42,740 DEBUG [RS:0;jenkins-hbase3:45997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-04T08:10:42,778 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-04T08:10:42,786 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-04T08:10:42,791 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-04T08:10:42,798 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,46181,1696407040117 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-04T08:10:42,804 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59883, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:10:42,805 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-04T08:10:42,806 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:10:42,810 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696407072810 2023-10-04T08:10:42,813 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-04T08:10:42,818 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46181 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-04T08:10:42,819 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-04T08:10:42,820 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-04T08:10:42,826 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,826 INFO [PEWorker-2 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-04T08:10:42,827 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-04T08:10:42,840 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-04T08:10:42,841 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-04T08:10:42,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-04T08:10:42,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-04T08:10:42,844 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-04T08:10:42,845 INFO [PEWorker-2 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-04T08:10:42,845 INFO [PEWorker-2 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:10:42,848 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-04T08:10:42,851 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-04T08:10:42,852 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-04T08:10:42,858 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-04T08:10:42,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-04T08:10:42,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696407042862,5,FailOnTimeoutGroup] 2023-10-04T08:10:42,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696407042866,5,FailOnTimeoutGroup] 2023-10-04T08:10:42,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-04T08:10:42,866 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-04T08:10:42,866 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-04T08:10:42,867 WARN [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 30000 ms and then retrying. 2023-10-04T08:10:42,867 DEBUG [PEWorker-2 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:10:42,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-04T08:10:42,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-04T08:10:42,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-04T08:10:42,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-04T08:10:42,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:10:42,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-04T08:10:42,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-04T08:10:42,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:10:42,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-04T08:10:42,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-04T08:10:42,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:10:42,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:10:42,882 DEBUG [PEWorker-2 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740 2023-10-04T08:10:42,883 DEBUG [PEWorker-2 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740 2023-10-04T08:10:42,886 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-04T08:10:42,888 DEBUG [PEWorker-2 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-04T08:10:42,895 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:10:42,896 INFO [PEWorker-2 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63191106, jitterRate=-0.058379143476486206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-04T08:10:42,900 DEBUG [PEWorker-2 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-04T08:10:42,900 DEBUG [PEWorker-2 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-04T08:10:42,900 INFO [PEWorker-2 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-04T08:10:42,900 DEBUG [PEWorker-2 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-04T08:10:42,900 DEBUG [PEWorker-2 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-04T08:10:42,901 DEBUG [PEWorker-2 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-04T08:10:42,903 INFO [PEWorker-2 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-04T08:10:42,903 DEBUG [PEWorker-2 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-04T08:10:42,908 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-04T08:10:42,908 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-04T08:10:42,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-04T08:10:42,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-04T08:10:42,937 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-04T08:10:43,088 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:44,088 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:45,089 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:46,090 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:47,090 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:48,091 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:49,091 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:50,092 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:51,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-04T08:10:51,032 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-04T08:10:51,093 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:52,093 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:53,094 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:54,094 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:55,095 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:56,095 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:57,096 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:58,097 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:10:59,097 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:00,098 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:01,098 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:02,099 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:03,099 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:04,100 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:05,100 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:06,101 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:07,102 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:08,102 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:09,103 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:10,103 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:11,104 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:12,104 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:12,709 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-04T08:11:12,713 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-04T08:11:12,868 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,46181,1696407040117 with isa=jenkins-hbase3.apache.org/172.31.12.81:45997, startcode=1696407041329 2023-10-04T08:11:12,874 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46181 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:12,886 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:11:12,886 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:45837 2023-10-04T08:11:12,886 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=-1 2023-10-04T08:11:12,892 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-10-04T08:11:12,893 DEBUG [RS:0;jenkins-hbase3:45997 {}] zookeeper.ZKUtil(111): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:12,893 WARN [RS:0;jenkins-hbase3:45997 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-04T08:11:12,893 INFO [RS:0;jenkins-hbase3:45997 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-04T08:11:12,893 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:12,896 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,45997,1696407041329] 2023-10-04T08:11:12,910 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-04T08:11:12,924 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-04T08:11:12,937 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-04T08:11:12,941 INFO [RS:0;jenkins-hbase3:45997 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-04T08:11:12,942 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,942 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-04T08:11:12,951 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,951 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,952 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,952 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,952 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,952 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,953 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-04T08:11:12,953 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,953 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,953 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,953 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,954 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-04T08:11:12,954 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-04T08:11:12,954 DEBUG [RS:0;jenkins-hbase3:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-04T08:11:12,955 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,955 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,955 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,955 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,955 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45997,1696407041329-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-04T08:11:12,977 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-04T08:11:12,980 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,45997,1696407041329-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:12,997 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,45997,1696407041329 started 2023-10-04T08:11:12,997 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,45997,1696407041329, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:45997, sessionid=0x1017c868d4b0001 2023-10-04T08:11:12,997 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-04T08:11:12,997 DEBUG [RS:0;jenkins-hbase3:45997 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:12,997 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45997,1696407041329' 2023-10-04T08:11:12,998 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2023-10-04T08:11:12,998 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,45997,1696407041329' 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2023-10-04T08:11:12,999 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2023-10-04T08:11:13,000 DEBUG [RS:0;jenkins-hbase3:45997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-04T08:11:13,000 INFO [RS:0;jenkins-hbase3:45997 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-04T08:11:13,000 INFO [RS:0;jenkins-hbase3:45997 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-04T08:11:13,105 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:14,106 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:15,106 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:16,107 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:17,107 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:18,108 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:19,108 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:20,109 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:21,110 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:22,110 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:23,111 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:24,111 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:25,112 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:26,112 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:27,113 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:28,114 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:29,114 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:30,115 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:31,115 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:32,116 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:33,116 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:34,117 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:35,118 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:36,118 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:37,119 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:38,119 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:39,120 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:40,120 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:41,121 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:42,122 WARN [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-10-04T08:11:43,008 INFO [RS:0;jenkins-hbase3:45997 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-04T08:11:43,014 INFO [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45997%2C1696407041329, suffix=, logDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329, archiveDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/oldWALs, maxLogs=32 2023-10-04T08:11:43,027 DEBUG [RS:0;jenkins-hbase3:45997 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329/jenkins-hbase3.apache.org%2C45997%2C1696407041329.1696407103016, exclude list is [], retry=0 2023-10-04T08:11:43,033 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK] 2023-10-04T08:11:43,040 INFO [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329/jenkins-hbase3.apache.org%2C45997%2C1696407041329.1696407103016 2023-10-04T08:11:43,041 DEBUG [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK]] 2023-10-04T08:11:43,123 DEBUG [jenkins-hbase3:46181 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-04T08:11:43,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:43,135 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45997,1696407041329, state=OPENING 2023-10-04T08:11:43,142 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-04T08:11:43,143 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:11:43,143 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:11:43,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-10-04T08:11:43,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-10-04T08:11:43,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:11:43,331 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:43,334 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-04T08:11:43,337 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-04T08:11:43,351 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-04T08:11:43,352 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-04T08:11:43,352 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-04T08:11:43,355 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C45997%2C1696407041329.meta, suffix=.meta, logDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329, archiveDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/oldWALs, maxLogs=32 2023-10-04T08:11:43,366 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329/jenkins-hbase3.apache.org%2C45997%2C1696407041329.meta.1696407103356.meta, exclude list is [], retry=0 2023-10-04T08:11:43,370 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK] 2023-10-04T08:11:43,376 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/WALs/jenkins-hbase3.apache.org,45997,1696407041329/jenkins-hbase3.apache.org%2C45997%2C1696407041329.meta.1696407103356.meta 2023-10-04T08:11:43,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:35837,DS-3852f686-69f4-4819-98e9-7894b62bf37a,DISK]] 2023-10-04T08:11:43,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:11:43,378 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-04T08:11:43,394 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-04T08:11:43,398 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-04T08:11:43,403 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-04T08:11:43,404 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:43,404 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-04T08:11:43,404 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-04T08:11:43,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-04T08:11:43,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-04T08:11:43,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:43,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:11:43,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-04T08:11:43,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-04T08:11:43,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:43,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:11:43,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-04T08:11:43,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-04T08:11:43,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:43,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-04T08:11:43,415 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740 2023-10-04T08:11:43,417 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740 2023-10-04T08:11:43,420 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-04T08:11:43,422 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-04T08:11:43,424 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63250831, jitterRate=-0.057489171624183655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-04T08:11:43,425 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-04T08:11:43,435 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696407103324 2023-10-04T08:11:43,452 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-04T08:11:43,453 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-04T08:11:43,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:43,456 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,45997,1696407041329, state=OPEN 2023-10-04T08:11:43,459 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-10-04T08:11:43,459 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2023-10-04T08:11:43,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-10-04T08:11:43,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2023-10-04T08:11:43,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-04T08:11:43,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,45997,1696407041329 in 312 msec 2023-10-04T08:11:43,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-04T08:11:43,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.545 sec 2023-10-04T08:11:43,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1 mins, 0.781 sec 2023-10-04T08:11:43,476 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696407103476, completionTime=-1 2023-10-04T08:11:43,476 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-04T08:11:43,476 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-04T08:11:43,541 DEBUG [hconnection-0x1fda5600-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:11:43,544 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48954, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:11:43,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-04T08:11:43,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696407163557 2023-10-04T08:11:43,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696407223557 2023-10-04T08:11:43,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 81 msec 2023-10-04T08:11:43,592 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46181,1696407040117-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:43,592 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46181,1696407040117-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:43,592 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46181,1696407040117-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:43,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:46181, period=300000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:43,595 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-04T08:11:43,602 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-04T08:11:43,614 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-04T08:11:43,615 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-04T08:11:43,624 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-04T08:11:43,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:11:43,628 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:43,630 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:11:43,649 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 015af7fd5d2067013019f6874c98f002, NAME => 'hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:11:44,060 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:44,061 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 015af7fd5d2067013019f6874c98f002, disabling compactions & flushes 2023-10-04T08:11:44,061 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,061 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,061 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. after waiting 0 ms 2023-10-04T08:11:44,061 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,061 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,061 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 015af7fd5d2067013019f6874c98f002: 2023-10-04T08:11:44,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:11:44,078 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696407104066"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407104066"}]},"ts":"1696407104066"} 2023-10-04T08:11:44,110 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:11:44,111 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:11:44,116 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407104112"}]},"ts":"1696407104112"} 2023-10-04T08:11:44,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-04T08:11:44,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=015af7fd5d2067013019f6874c98f002, ASSIGN}] 2023-10-04T08:11:44,133 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=015af7fd5d2067013019f6874c98f002, ASSIGN 2023-10-04T08:11:44,135 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=015af7fd5d2067013019f6874c98f002, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:11:44,286 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=015af7fd5d2067013019f6874c98f002, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:44,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 015af7fd5d2067013019f6874c98f002, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:11:44,452 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,452 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 015af7fd5d2067013019f6874c98f002, NAME => 'hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:11:44,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:44,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,454 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,456 INFO [StoreOpener-015af7fd5d2067013019f6874c98f002-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,458 INFO [StoreOpener-015af7fd5d2067013019f6874c98f002-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 015af7fd5d2067013019f6874c98f002 columnFamilyName info 2023-10-04T08:11:44,458 DEBUG [StoreOpener-015af7fd5d2067013019f6874c98f002-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:44,459 INFO [StoreOpener-015af7fd5d2067013019f6874c98f002-1 {}] regionserver.HStore(324): Store=015af7fd5d2067013019f6874c98f002/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:11:44,460 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,461 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,465 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 015af7fd5d2067013019f6874c98f002 2023-10-04T08:11:44,468 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:11:44,469 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 015af7fd5d2067013019f6874c98f002; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72438409, jitterRate=0.07941640913486481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:11:44,470 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 015af7fd5d2067013019f6874c98f002: 2023-10-04T08:11:44,472 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002., pid=6, masterSystemTime=1696407104445 2023-10-04T08:11:44,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,476 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:11:44,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=015af7fd5d2067013019f6874c98f002, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:44,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-04T08:11:44,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 015af7fd5d2067013019f6874c98f002, server=jenkins-hbase3.apache.org,45997,1696407041329 in 190 msec 2023-10-04T08:11:44,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-04T08:11:44,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=015af7fd5d2067013019f6874c98f002, ASSIGN in 355 msec 2023-10-04T08:11:44,491 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:11:44,491 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407104491"}]},"ts":"1696407104491"} 2023-10-04T08:11:44,495 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-04T08:11:44,498 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:11:44,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 882 msec 2023-10-04T08:11:44,527 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2023-10-04T08:11:44,529 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2023-10-04T08:11:44,529 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:11:44,529 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:11:44,573 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-04T08:11:44,598 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-10-04T08:11:44,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 37 msec 2023-10-04T08:11:44,612 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-04T08:11:44,626 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2023-10-04T08:11:44,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-10-04T08:11:44,639 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2023-10-04T08:11:44,642 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2023-10-04T08:11:44,642 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 63.237sec 2023-10-04T08:11:44,644 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-04T08:11:44,647 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-04T08:11:44,648 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-04T08:11:44,649 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-04T08:11:44,649 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-04T08:11:44,652 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46181,1696407040117-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-04T08:11:44,652 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46181,1696407040117-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-04T08:11:44,664 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-04T08:11:44,665 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-04T08:11:44,727 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47dbb7dd to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:11:44,736 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6575f0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:11:44,740 DEBUG [Listener at localhost/43533 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-04T08:11:44,740 DEBUG [Listener at localhost/43533 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-04T08:11:44,754 DEBUG [hconnection-0x25edebaf-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:11:44,765 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48956, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:11:44,776 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:11:44,798 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=195, OpenFileDescriptor=480, MaxFileDescriptor=60000, SystemLoadAverage=124, ProcessCount=172, AvailableMemoryMB=6478 2023-10-04T08:11:44,802 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x61dd1f7d to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:11:44,814 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x61dd1f7d {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e73a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:11:44,819 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6a67482e to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:11:44,823 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64feacff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:11:44,833 DEBUG [Listener at localhost/43533 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-04T08:11:44,835 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60980, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-04T08:11:44,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:11:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2023-10-04T08:11:44,853 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:11:44,854 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:44,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:11:44,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2023-10-04T08:11:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-04T08:11:45,274 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9c208f35f121b8b6fe14f4139d19c72e, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:11:45,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:45,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 9c208f35f121b8b6fe14f4139d19c72e, disabling compactions & flushes 2023-10-04T08:11:45,688 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:45,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:45,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. after waiting 0 ms 2023-10-04T08:11:45,688 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:45,689 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:45,689 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9c208f35f121b8b6fe14f4139d19c72e: 2023-10-04T08:11:45,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:11:45,691 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1696407105691"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407105691"}]},"ts":"1696407105691"} 2023-10-04T08:11:45,695 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:11:45,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:11:45,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407105697"}]},"ts":"1696407105697"} 2023-10-04T08:11:45,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:11:45,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=9c208f35f121b8b6fe14f4139d19c72e, ASSIGN}] 2023-10-04T08:11:45,707 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=9c208f35f121b8b6fe14f4139d19c72e, ASSIGN 2023-10-04T08:11:45,708 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=9c208f35f121b8b6fe14f4139d19c72e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:11:45,859 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9c208f35f121b8b6fe14f4139d19c72e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:45,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 9c208f35f121b8b6fe14f4139d19c72e, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:11:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-04T08:11:46,024 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:46,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7254): Opening region: {ENCODED => 9c208f35f121b8b6fe14f4139d19c72e, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:11:46,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:46,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7296): checking encryption for 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7299): checking classloading for 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,027 INFO [StoreOpener-9c208f35f121b8b6fe14f4139d19c72e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,029 INFO [StoreOpener-9c208f35f121b8b6fe14f4139d19c72e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c208f35f121b8b6fe14f4139d19c72e columnFamilyName testFamily 2023-10-04T08:11:46,030 DEBUG [StoreOpener-9c208f35f121b8b6fe14f4139d19c72e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:46,031 INFO [StoreOpener-9c208f35f121b8b6fe14f4139d19c72e-1 {}] regionserver.HStore(324): Store=9c208f35f121b8b6fe14f4139d19c72e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:11:46,032 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,033 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,036 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1076): writing seq id for 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:11:46,040 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:11:46,041 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): Opened 9c208f35f121b8b6fe14f4139d19c72e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72545787, jitterRate=0.08101646602153778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:11:46,042 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(992): Region open journal for 9c208f35f121b8b6fe14f4139d19c72e: 2023-10-04T08:11:46,044 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e., pid=11, masterSystemTime=1696407106017 2023-10-04T08:11:46,047 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:46,047 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:11:46,048 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=9c208f35f121b8b6fe14f4139d19c72e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:46,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=11, resume processing ppid=10 2023-10-04T08:11:46,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 9c208f35f121b8b6fe14f4139d19c72e, server=jenkins-hbase3.apache.org,45997,1696407041329 in 188 msec 2023-10-04T08:11:46,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=10, resume processing ppid=9 2023-10-04T08:11:46,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=9c208f35f121b8b6fe14f4139d19c72e, ASSIGN in 349 msec 2023-10-04T08:11:46,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:11:46,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407106059"}]},"ts":"1696407106059"} 2023-10-04T08:11:46,063 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:11:46,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:11:46,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 1.2170 sec 2023-10-04T08:11:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-04T08:11:47,885 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2023-10-04T08:11:47,885 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:11:47,887 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:11:47,891 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2023-10-04T08:11:47,891 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:11:47,891 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2023-10-04T08:11:47,894 DEBUG [hconnection-0x3b5b5eb1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:11:47,896 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48962, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:11:47,911 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:11:47,917 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-10-04T08:11:47,937 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:11:47,939 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x61dd1f7d {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:11:47,946 DEBUG [Listener at localhost/43533 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:11:47,949 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48976, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:11:47,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:11:47,962 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-10-04T08:11:47,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885131 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,965 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-10-04T08:11:47,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885131 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666485 2023-10-04T08:11:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48976 deadline: 1696407110964, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:11:47,968 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885131 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:11:47,970 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-10-04T08:11:47,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885132 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,971 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-10-04T08:11:47,972 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:11:47,973 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:11:47,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885133 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666483 2023-10-04T08:11:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48976 deadline: 1696407110972, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:11:47,975 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885133 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:11:47,976 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:11:47,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885134 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-452929323842666482 2023-10-04T08:11:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48976 deadline: 1696407110975, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:11:47,978 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885134 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:11:47,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885135 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:11:47,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-452929323842666481 2023-10-04T08:11:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48976 deadline: 1696407110979, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:11:47,983 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:11:47,984 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885135 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:11:47,985 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:11:47,987 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6a67482e to 127.0.0.1:50391 2023-10-04T08:11:47,987 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:11:47,988 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:11:47,988 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:11:47,996 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:11:47,997 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x61dd1f7d to 127.0.0.1:50391 2023-10-04T08:11:48,010 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=203 (was 195) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1534050870_18 at /127.0.0.1:37676 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x61dd1f7d-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x25edebaf-shared-pool-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x6a67482e java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$438/1381331103.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x6a67482e-SendThread(127.0.0.1:50391) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x61dd1f7d-SendThread(127.0.0.1:50391) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: process reaper sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x61dd1f7d java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$438/1381331103.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.lang.Thread.sleep(Native Method) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:600) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:496) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:50391@0x6a67482e-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=492 (was 480) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=122 (was 124), ProcessCount=172 (was 172), AvailableMemoryMB=6455 (was 6478) 2023-10-04T08:11:48,021 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=203, OpenFileDescriptor=492, MaxFileDescriptor=60000, SystemLoadAverage=122, ProcessCount=172, AvailableMemoryMB=6454 2023-10-04T08:11:48,023 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x13c436ce to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:11:48,028 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x13c436ce {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f04fc68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:11:48,030 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e616203 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:11:48,034 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e3b4c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:11:48,037 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:11:48,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2023-10-04T08:11:48,041 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:11:48,041 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:48,041 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2023-10-04T08:11:48,043 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:11:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-04T08:11:48,462 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7280776114ac4274bfa1c2878b754e7e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 7280776114ac4274bfa1c2878b754e7e, disabling compactions & flushes 2023-10-04T08:11:48,876 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. after waiting 0 ms 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:48,876 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:48,876 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7280776114ac4274bfa1c2878b754e7e: 2023-10-04T08:11:48,879 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:11:48,879 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1696407108879"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407108879"}]},"ts":"1696407108879"} 2023-10-04T08:11:48,882 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:11:48,883 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:11:48,884 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407108883"}]},"ts":"1696407108883"} 2023-10-04T08:11:48,886 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:11:48,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7280776114ac4274bfa1c2878b754e7e, ASSIGN}] 2023-10-04T08:11:48,892 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7280776114ac4274bfa1c2878b754e7e, ASSIGN 2023-10-04T08:11:48,893 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7280776114ac4274bfa1c2878b754e7e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:11:48,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:11:48,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-04T08:11:48,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2023-10-04T08:11:48,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-04T08:11:49,044 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=7280776114ac4274bfa1c2878b754e7e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-04T08:11:49,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 7280776114ac4274bfa1c2878b754e7e, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:11:49,207 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:49,207 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7254): Opening region: {ENCODED => 7280776114ac4274bfa1c2878b754e7e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:11:49,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:11:49,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7296): checking encryption for 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,208 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7299): checking classloading for 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,210 INFO [StoreOpener-7280776114ac4274bfa1c2878b754e7e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,212 INFO [StoreOpener-7280776114ac4274bfa1c2878b754e7e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7280776114ac4274bfa1c2878b754e7e columnFamilyName testFamily 2023-10-04T08:11:49,213 DEBUG [StoreOpener-7280776114ac4274bfa1c2878b754e7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:11:49,214 INFO [StoreOpener-7280776114ac4274bfa1c2878b754e7e-1 {}] regionserver.HStore(324): Store=7280776114ac4274bfa1c2878b754e7e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:11:49,215 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,220 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1076): writing seq id for 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:11:49,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:11:49,225 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1093): Opened 7280776114ac4274bfa1c2878b754e7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69522032, jitterRate=0.03595900535583496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:11:49,226 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(992): Region open journal for 7280776114ac4274bfa1c2878b754e7e: 2023-10-04T08:11:49,228 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e., pid=14, masterSystemTime=1696407109200 2023-10-04T08:11:49,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:49,231 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:11:49,232 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=7280776114ac4274bfa1c2878b754e7e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:11:49,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-04T08:11:49,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 7280776114ac4274bfa1c2878b754e7e, server=jenkins-hbase3.apache.org,45997,1696407041329 in 187 msec 2023-10-04T08:11:49,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=13, resume processing ppid=12 2023-10-04T08:11:49,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=7280776114ac4274bfa1c2878b754e7e, ASSIGN in 347 msec 2023-10-04T08:11:49,241 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:11:49,242 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407109241"}]},"ts":"1696407109241"} 2023-10-04T08:11:49,244 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:11:49,248 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:11:49,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 1.2110 sec 2023-10-04T08:11:51,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2023-10-04T08:11:51,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:11:51,031 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-04T08:11:51,031 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-04T08:11:51,033 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-04T08:11:51,033 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-04T08:11:51,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-04T08:11:51,034 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-04T08:11:51,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2023-10-04T08:11:51,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:11:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-04T08:11:51,049 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2023-10-04T08:11:51,049 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:11:51,049 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:11:51,054 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2023-10-04T08:11:51,054 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:11:51,054 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2023-10-04T08:11:51,056 DEBUG [hconnection-0xf0f76ff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:11:51,060 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:11:51,073 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:11:51,077 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:11:51,078 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:11:54,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:11:54,995 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2023-10-04T08:11:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 89 bytes 2023-10-04T08:11:58,100 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:12:00,080 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:12:00,081 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-10-04T08:12:00.081Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11004: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1978ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:12:00,083 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2023-10-04T08:12:00,083 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e616203 to 127.0.0.1:50391 2023-10-04T08:12:00,084 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:00,084 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:00,084 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:00,084 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:00,084 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x13c436ce to 127.0.0.1:50391 2023-10-04T08:12:00,097 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=200 (was 203), OpenFileDescriptor=486 (was 492), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=119 (was 122), ProcessCount=172 (was 172), AvailableMemoryMB=6410 (was 6454) 2023-10-04T08:12:00,109 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=200, OpenFileDescriptor=486, MaxFileDescriptor=60000, SystemLoadAverage=119, ProcessCount=172, AvailableMemoryMB=6410 2023-10-04T08:12:00,113 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x37c5512d to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:00,118 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x37c5512d {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d265852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:00,120 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5edbb9c8 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:00,124 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56dd5b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:00,128 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:00,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2023-10-04T08:12:00,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:00,131 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:00,131 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2023-10-04T08:12:00,132 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=15 2023-10-04T08:12:00,550 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8b44dd21679fb8b47ec68172b7a9d6ab, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 8b44dd21679fb8b47ec68172b7a9d6ab, disabling compactions & flushes 2023-10-04T08:12:00,561 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. after waiting 0 ms 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,561 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,561 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8b44dd21679fb8b47ec68172b7a9d6ab: 2023-10-04T08:12:00,563 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:00,563 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1696407120563"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407120563"}]},"ts":"1696407120563"} 2023-10-04T08:12:00,566 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:00,567 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:00,567 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407120567"}]},"ts":"1696407120567"} 2023-10-04T08:12:00,569 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:00,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=8b44dd21679fb8b47ec68172b7a9d6ab, ASSIGN}] 2023-10-04T08:12:00,576 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=8b44dd21679fb8b47ec68172b7a9d6ab, ASSIGN 2023-10-04T08:12:00,577 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=8b44dd21679fb8b47ec68172b7a9d6ab, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:00,728 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=8b44dd21679fb8b47ec68172b7a9d6ab, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:00,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 8b44dd21679fb8b47ec68172b7a9d6ab, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:00,892 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7254): Opening region: {ENCODED => 8b44dd21679fb8b47ec68172b7a9d6ab, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:00,892 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:00,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7296): checking encryption for 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7299): checking classloading for 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,895 INFO [StoreOpener-8b44dd21679fb8b47ec68172b7a9d6ab-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,897 INFO [StoreOpener-8b44dd21679fb8b47ec68172b7a9d6ab-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b44dd21679fb8b47ec68172b7a9d6ab columnFamilyName testFamily 2023-10-04T08:12:00,897 DEBUG [StoreOpener-8b44dd21679fb8b47ec68172b7a9d6ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:00,898 INFO [StoreOpener-8b44dd21679fb8b47ec68172b7a9d6ab-1 {}] regionserver.HStore(324): Store=8b44dd21679fb8b47ec68172b7a9d6ab/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:00,899 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,900 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1076): writing seq id for 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:12:00,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:00,908 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1093): Opened 8b44dd21679fb8b47ec68172b7a9d6ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63959748, jitterRate=-0.046925485134124756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:00,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(992): Region open journal for 8b44dd21679fb8b47ec68172b7a9d6ab: 2023-10-04T08:12:00,910 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab., pid=17, masterSystemTime=1696407120884 2023-10-04T08:12:00,913 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,913 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:12:00,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=8b44dd21679fb8b47ec68172b7a9d6ab, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:00,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-04T08:12:00,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 8b44dd21679fb8b47ec68172b7a9d6ab, server=jenkins-hbase3.apache.org,45997,1696407041329 in 185 msec 2023-10-04T08:12:00,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=16, resume processing ppid=15 2023-10-04T08:12:00,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=8b44dd21679fb8b47ec68172b7a9d6ab, ASSIGN in 345 msec 2023-10-04T08:12:00,923 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:00,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407120924"}]},"ts":"1696407120924"} 2023-10-04T08:12:00,926 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:00,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:00,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 801 msec 2023-10-04T08:12:00,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2023-10-04T08:12:01,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2023-10-04T08:12:01,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=15 2023-10-04T08:12:01,136 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2023-10-04T08:12:01,136 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:01,136 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:01,140 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:01,140 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:01,141 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2023-10-04T08:12:01,143 DEBUG [hconnection-0x747b097d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:01,146 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55440, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:01,158 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:01,161 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:01,161 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:12:01,163 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x37c5512d {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:12:01,163 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x37c5512d {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:01,165 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55446, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:12:01,169 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:12:01,170 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885147 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:01,171 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:12:01,171 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885147 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:01,172 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-10-04T08:12:04,601 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:06,462 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:10,180 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2023-10-04T08:12:10,180 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885147 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:12:10,180 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5edbb9c8 to 127.0.0.1:50391 2023-10-04T08:12:10,181 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:10,181 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:10,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666469' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-04T08:12:10,181 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:10,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:55446 deadline: 1696407136180, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666469' does not exist 2023-10-04T08:12:10,181 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:10,181 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x37c5512d to 127.0.0.1:50391 2023-10-04T08:12:10,182 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,45997,1696407041329 for closing scanner id = -452929323842666469 for 8b44dd21679fb8b47ec68172b7a9d6ab of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-10-04T08:12:10,194 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=204 (was 200) - Thread LEAK? -, OpenFileDescriptor=487 (was 486) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=115 (was 119), ProcessCount=172 (was 172), AvailableMemoryMB=6330 (was 6410) 2023-10-04T08:12:10,203 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=204, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=115, ProcessCount=172, AvailableMemoryMB=6330 2023-10-04T08:12:10,206 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2ccaff0f to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:10,210 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x2ccaff0f {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75161231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:10,211 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0236c95e to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:10,216 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e9264e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:10,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2023-10-04T08:12:10,223 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:10,223 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:10,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2023-10-04T08:12:10,224 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-10-04T08:12:10,637 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 051899d47523a890d1ad457722c3b97f, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:10,672 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 051899d47523a890d1ad457722c3b97f, disabling compactions & flushes 2023-10-04T08:12:11,049 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. after waiting 0 ms 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,049 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 051899d47523a890d1ad457722c3b97f: 2023-10-04T08:12:11,051 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:11,051 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1696407131051"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407131051"}]},"ts":"1696407131051"} 2023-10-04T08:12:11,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:11,055 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:11,055 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407131055"}]},"ts":"1696407131055"} 2023-10-04T08:12:11,057 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:11,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=051899d47523a890d1ad457722c3b97f, ASSIGN}] 2023-10-04T08:12:11,064 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=051899d47523a890d1ad457722c3b97f, ASSIGN 2023-10-04T08:12:11,065 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=051899d47523a890d1ad457722c3b97f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:11,216 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=051899d47523a890d1ad457722c3b97f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:11,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 051899d47523a890d1ad457722c3b97f, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-10-04T08:12:11,378 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,378 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7254): Opening region: {ENCODED => 051899d47523a890d1ad457722c3b97f, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:11,378 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:11,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7296): checking encryption for 051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7299): checking classloading for 051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,381 INFO [StoreOpener-051899d47523a890d1ad457722c3b97f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,383 INFO [StoreOpener-051899d47523a890d1ad457722c3b97f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 051899d47523a890d1ad457722c3b97f columnFamilyName testFamily 2023-10-04T08:12:11,383 DEBUG [StoreOpener-051899d47523a890d1ad457722c3b97f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:11,384 INFO [StoreOpener-051899d47523a890d1ad457722c3b97f-1 {}] regionserver.HStore(324): Store=051899d47523a890d1ad457722c3b97f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:11,385 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,385 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,389 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1076): writing seq id for 051899d47523a890d1ad457722c3b97f 2023-10-04T08:12:11,392 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:11,393 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): Opened 051899d47523a890d1ad457722c3b97f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68066766, jitterRate=0.014273852109909058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:11,394 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(992): Region open journal for 051899d47523a890d1ad457722c3b97f: 2023-10-04T08:12:11,395 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f., pid=20, masterSystemTime=1696407131371 2023-10-04T08:12:11,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,397 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:12:11,398 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=051899d47523a890d1ad457722c3b97f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:11,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-04T08:12:11,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 051899d47523a890d1ad457722c3b97f, server=jenkins-hbase3.apache.org,45997,1696407041329 in 182 msec 2023-10-04T08:12:11,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=19, resume processing ppid=18 2023-10-04T08:12:11,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=051899d47523a890d1ad457722c3b97f, ASSIGN in 342 msec 2023-10-04T08:12:11,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:11,407 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407131407"}]},"ts":"1696407131407"} 2023-10-04T08:12:11,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:11,412 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:11,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.1920 sec 2023-10-04T08:12:12,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2023-10-04T08:12:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=18 2023-10-04T08:12:13,229 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2023-10-04T08:12:13,229 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:13,229 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:13,233 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:13,234 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:13,234 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2023-10-04T08:12:13,236 DEBUG [hconnection-0x4e4e0f23-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:13,239 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46226, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:13,248 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:13,251 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-10-04T08:12:13,252 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-10-04T08:12:13,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885151 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,254 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-10-04T08:12:13,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885151 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666465 2023-10-04T08:12:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:46226 deadline: 1696407134255, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:12:13,256 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:13,259 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-10-04T08:12:13,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885152 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,260 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-10-04T08:12:13,260 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:12:13,261 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:12:13,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885153 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666463 2023-10-04T08:12:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:46226 deadline: 1696407134262, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:12:13,263 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:13,265 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:12:13,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885154 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-452929323842666462 2023-10-04T08:12:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:46226 deadline: 1696407134265, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:12:13,267 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:13,269 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:12:13,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885155 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:13,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-452929323842666461 2023-10-04T08:12:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:46226 deadline: 1696407134269, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:12:13,270 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:13,272 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:12:13,273 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0236c95e to 127.0.0.1:50391 2023-10-04T08:12:13,273 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:13,273 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:13,273 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:13,273 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:13,274 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2ccaff0f to 127.0.0.1:50391 2023-10-04T08:12:13,287 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=207 (was 204) - Thread LEAK? -, OpenFileDescriptor=489 (was 487) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=114 (was 115), ProcessCount=172 (was 172), AvailableMemoryMB=6326 (was 6330) 2023-10-04T08:12:13,299 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=207, OpenFileDescriptor=489, MaxFileDescriptor=60000, SystemLoadAverage=114, ProcessCount=172, AvailableMemoryMB=6326 2023-10-04T08:12:13,301 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x31cd2359 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:13,305 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x31cd2359 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15736c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:13,307 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7823c634 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:13,311 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f93309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:13,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2023-10-04T08:12:13,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:13,317 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:13,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2023-10-04T08:12:13,318 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-10-04T08:12:13,732 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b1b68c5387402f2415aa871d52ff711f, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing b1b68c5387402f2415aa871d52ff711f, disabling compactions & flushes 2023-10-04T08:12:14,144 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. after waiting 0 ms 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,144 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,144 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for b1b68c5387402f2415aa871d52ff711f: 2023-10-04T08:12:14,147 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:14,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1696407134147"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407134147"}]},"ts":"1696407134147"} 2023-10-04T08:12:14,149 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:14,150 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:14,151 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407134150"}]},"ts":"1696407134150"} 2023-10-04T08:12:14,153 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:14,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=b1b68c5387402f2415aa871d52ff711f, ASSIGN}] 2023-10-04T08:12:14,161 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=b1b68c5387402f2415aa871d52ff711f, ASSIGN 2023-10-04T08:12:14,162 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=b1b68c5387402f2415aa871d52ff711f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:14,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=b1b68c5387402f2415aa871d52ff711f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:14,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure b1b68c5387402f2415aa871d52ff711f, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-10-04T08:12:14,475 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7254): Opening region: {ENCODED => b1b68c5387402f2415aa871d52ff711f, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:14,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:14,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7296): checking encryption for b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,476 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7299): checking classloading for b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,478 INFO [StoreOpener-b1b68c5387402f2415aa871d52ff711f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,481 INFO [StoreOpener-b1b68c5387402f2415aa871d52ff711f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1b68c5387402f2415aa871d52ff711f columnFamilyName testFamily 2023-10-04T08:12:14,482 DEBUG [StoreOpener-b1b68c5387402f2415aa871d52ff711f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:14,482 INFO [StoreOpener-b1b68c5387402f2415aa871d52ff711f-1 {}] regionserver.HStore(324): Store=b1b68c5387402f2415aa871d52ff711f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:14,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,484 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1076): writing seq id for b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:12:14,490 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:14,491 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): Opened b1b68c5387402f2415aa871d52ff711f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62008892, jitterRate=-0.07599550485610962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:14,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(992): Region open journal for b1b68c5387402f2415aa871d52ff711f: 2023-10-04T08:12:14,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f., pid=23, masterSystemTime=1696407134469 2023-10-04T08:12:14,496 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,496 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:12:14,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=b1b68c5387402f2415aa871d52ff711f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:14,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-04T08:12:14,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure b1b68c5387402f2415aa871d52ff711f, server=jenkins-hbase3.apache.org,45997,1696407041329 in 184 msec 2023-10-04T08:12:14,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=22, resume processing ppid=21 2023-10-04T08:12:14,505 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=b1b68c5387402f2415aa871d52ff711f, ASSIGN in 342 msec 2023-10-04T08:12:14,506 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:14,506 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407134506"}]},"ts":"1696407134506"} 2023-10-04T08:12:14,508 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:14,511 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:14,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.1980 sec 2023-10-04T08:12:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=21 2023-10-04T08:12:16,322 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2023-10-04T08:12:16,322 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:16,323 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:16,326 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:16,326 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:16,327 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2023-10-04T08:12:16,329 DEBUG [hconnection-0x4d37913-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:16,332 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46242, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:16,341 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:16,343 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:16,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:16,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-10-04T08:12:16,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 12 bytes 2023-10-04T08:12:18,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:46242 deadline: 1696407139345, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:18,361 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:18,362 DEBUG [Listener at localhost/43533 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885160 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:12:18,363 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2019 2023-10-04T08:12:18,364 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7823c634 to 127.0.0.1:50391 2023-10-04T08:12:18,364 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:18,364 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:18,364 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:18,364 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:18,364 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x31cd2359 to 127.0.0.1:50391 2023-10-04T08:12:18,377 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=208 (was 207) - Thread LEAK? -, OpenFileDescriptor=492 (was 489) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=112 (was 114), ProcessCount=172 (was 172), AvailableMemoryMB=6324 (was 6326) 2023-10-04T08:12:18,387 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=208, OpenFileDescriptor=492, MaxFileDescriptor=60000, SystemLoadAverage=112, ProcessCount=172, AvailableMemoryMB=6323 2023-10-04T08:12:18,389 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x33d33e6c to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:18,394 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x33d33e6c {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51c306e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:18,396 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f6bda88 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:18,402 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298d5e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:18,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2023-10-04T08:12:18,407 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:18,407 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:18,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2023-10-04T08:12:18,408 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-10-04T08:12:18,822 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 96dd4b21100c44df0f1d2c389effacfe, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:18,999 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2023-10-04T08:12:19,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:19,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 96dd4b21100c44df0f1d2c389effacfe, disabling compactions & flushes 2023-10-04T08:12:19,233 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. after waiting 0 ms 2023-10-04T08:12:19,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,234 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 96dd4b21100c44df0f1d2c389effacfe: 2023-10-04T08:12:19,235 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:19,236 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1696407139235"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407139235"}]},"ts":"1696407139235"} 2023-10-04T08:12:19,238 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:19,239 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:19,239 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407139239"}]},"ts":"1696407139239"} 2023-10-04T08:12:19,241 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:19,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=96dd4b21100c44df0f1d2c389effacfe, ASSIGN}] 2023-10-04T08:12:19,248 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=96dd4b21100c44df0f1d2c389effacfe, ASSIGN 2023-10-04T08:12:19,249 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=96dd4b21100c44df0f1d2c389effacfe, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:19,400 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=96dd4b21100c44df0f1d2c389effacfe, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:19,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 96dd4b21100c44df0f1d2c389effacfe, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-10-04T08:12:19,560 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7254): Opening region: {ENCODED => 96dd4b21100c44df0f1d2c389effacfe, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:19,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:19,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7296): checking encryption for 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7299): checking classloading for 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,564 INFO [StoreOpener-96dd4b21100c44df0f1d2c389effacfe-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,566 INFO [StoreOpener-96dd4b21100c44df0f1d2c389effacfe-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96dd4b21100c44df0f1d2c389effacfe columnFamilyName testFamily 2023-10-04T08:12:19,567 DEBUG [StoreOpener-96dd4b21100c44df0f1d2c389effacfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:19,567 INFO [StoreOpener-96dd4b21100c44df0f1d2c389effacfe-1 {}] regionserver.HStore(324): Store=96dd4b21100c44df0f1d2c389effacfe/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:19,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,569 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1076): writing seq id for 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:12:19,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:19,576 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): Opened 96dd4b21100c44df0f1d2c389effacfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63016775, jitterRate=-0.06097687780857086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:19,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(992): Region open journal for 96dd4b21100c44df0f1d2c389effacfe: 2023-10-04T08:12:19,578 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., pid=26, masterSystemTime=1696407139555 2023-10-04T08:12:19,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,580 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:12:19,581 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=96dd4b21100c44df0f1d2c389effacfe, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:19,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=26, resume processing ppid=25 2023-10-04T08:12:19,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 96dd4b21100c44df0f1d2c389effacfe, server=jenkins-hbase3.apache.org,45997,1696407041329 in 180 msec 2023-10-04T08:12:19,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=25, resume processing ppid=24 2023-10-04T08:12:19,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=96dd4b21100c44df0f1d2c389effacfe, ASSIGN in 339 msec 2023-10-04T08:12:19,588 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:19,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407139588"}]},"ts":"1696407139588"} 2023-10-04T08:12:19,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:19,593 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:19,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 1.1890 sec 2023-10-04T08:12:21,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2023-10-04T08:12:21,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:21,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2023-10-04T08:12:21,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:21,031 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2023-10-04T08:12:21,031 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=24 2023-10-04T08:12:21,414 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2023-10-04T08:12:21,414 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:21,414 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:21,418 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:21,418 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:21,418 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2023-10-04T08:12:21,420 DEBUG [hconnection-0x25b014f5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:21,423 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50586, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:21,432 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:21,435 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:12:21,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:12:22,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 89 bytes 2023-10-04T08:12:23,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:12:24,440 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:12:24,441 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-10-04T08:12:24.441Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5012: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=986ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:12:24,442 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3007 2023-10-04T08:12:24,442 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f6bda88 to 127.0.0.1:50391 2023-10-04T08:12:24,442 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:24,442 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:24,442 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:24,443 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:24,443 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x33d33e6c to 127.0.0.1:50391 2023-10-04T08:12:24,455 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=207 (was 208), OpenFileDescriptor=490 (was 492), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=112 (was 112), ProcessCount=172 (was 172), AvailableMemoryMB=6319 (was 6323) 2023-10-04T08:12:24,465 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=207, OpenFileDescriptor=490, MaxFileDescriptor=60000, SystemLoadAverage=112, ProcessCount=172, AvailableMemoryMB=6318 2023-10-04T08:12:24,467 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c01cdf1 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:24,471 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x0c01cdf1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ccf5d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:24,473 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11a0d1d6 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:24,477 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3380f902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:24,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2023-10-04T08:12:24,482 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:24,482 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:24,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2023-10-04T08:12:24,483 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-10-04T08:12:24,667 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-04T08:12:24,667 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-04T08:12:24,898 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 999353f1059233c76a4049e6624f9aa5, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:24,951 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:25,063 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:25,088 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 999353f1059233c76a4049e6624f9aa5, disabling compactions & flushes 2023-10-04T08:12:25,309 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. after waiting 0 ms 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,309 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,309 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 999353f1059233c76a4049e6624f9aa5: 2023-10-04T08:12:25,311 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:25,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1696407145311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407145311"}]},"ts":"1696407145311"} 2023-10-04T08:12:25,314 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:25,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:25,315 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407145315"}]},"ts":"1696407145315"} 2023-10-04T08:12:25,317 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:25,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=999353f1059233c76a4049e6624f9aa5, ASSIGN}] 2023-10-04T08:12:25,323 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=999353f1059233c76a4049e6624f9aa5, ASSIGN 2023-10-04T08:12:25,324 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=999353f1059233c76a4049e6624f9aa5, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:25,475 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=999353f1059233c76a4049e6624f9aa5, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:25,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 999353f1059233c76a4049e6624f9aa5, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-10-04T08:12:25,636 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,636 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 999353f1059233c76a4049e6624f9aa5, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:25,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:25,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,639 INFO [StoreOpener-999353f1059233c76a4049e6624f9aa5-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,640 INFO [StoreOpener-999353f1059233c76a4049e6624f9aa5-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 999353f1059233c76a4049e6624f9aa5 columnFamilyName testFamily 2023-10-04T08:12:25,641 DEBUG [StoreOpener-999353f1059233c76a4049e6624f9aa5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:25,641 INFO [StoreOpener-999353f1059233c76a4049e6624f9aa5-1 {}] regionserver.HStore(324): Store=999353f1059233c76a4049e6624f9aa5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:25,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:12:25,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:25,650 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 999353f1059233c76a4049e6624f9aa5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72265534, jitterRate=0.0768403708934784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:25,651 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 999353f1059233c76a4049e6624f9aa5: 2023-10-04T08:12:25,652 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5., pid=29, masterSystemTime=1696407145631 2023-10-04T08:12:25,654 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,654 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:12:25,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=999353f1059233c76a4049e6624f9aa5, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:25,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=28 2023-10-04T08:12:25,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 999353f1059233c76a4049e6624f9aa5, server=jenkins-hbase3.apache.org,45997,1696407041329 in 179 msec 2023-10-04T08:12:25,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-04T08:12:25,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=999353f1059233c76a4049e6624f9aa5, ASSIGN in 338 msec 2023-10-04T08:12:25,662 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:25,662 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407145662"}]},"ts":"1696407145662"} 2023-10-04T08:12:25,664 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:25,666 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:25,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.1880 sec 2023-10-04T08:12:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=27 2023-10-04T08:12:27,488 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2023-10-04T08:12:27,488 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:27,488 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:27,493 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:27,493 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:27,493 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2023-10-04T08:12:27,495 DEBUG [hconnection-0x3e3bb722-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:27,498 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50598, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:27,508 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:27,511 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:27,511 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:12:27,513 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x0c01cdf1 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:12:27,513 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x0c01cdf1 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:27,515 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50610, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:12:27,521 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:12:27,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885172 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:27,522 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:12:27,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885172 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:27,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-10-04T08:12:30,530 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3007 2023-10-04T08:12:30,530 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11a0d1d6 to 127.0.0.1:50391 2023-10-04T08:12:30,530 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:30,530 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885172 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:12:30,530 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:30,530 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:30,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666444' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-04T08:12:30,531 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:30,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:50610 deadline: 1696407151530, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666444' does not exist 2023-10-04T08:12:30,531 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c01cdf1 to 127.0.0.1:50391 2023-10-04T08:12:30,531 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:30,531 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,45997,1696407041329 for closing scanner id = -452929323842666444 for 999353f1059233c76a4049e6624f9aa5 of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-10-04T08:12:30,543 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=209 (was 207) - Thread LEAK? -, OpenFileDescriptor=487 (was 490), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=119 (was 112) - SystemLoadAverage LEAK? -, ProcessCount=172 (was 172), AvailableMemoryMB=6318 (was 6318) 2023-10-04T08:12:30,554 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=209, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=119, ProcessCount=172, AvailableMemoryMB=6318 2023-10-04T08:12:30,556 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c47ca36 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:30,560 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x4c47ca36 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e290b6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:30,562 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e12bbc1 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:30,566 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b72d69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:30,569 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:30,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2023-10-04T08:12:30,572 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:30,572 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:30,572 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2023-10-04T08:12:30,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-10-04T08:12:30,985 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d8089a176acb4ff8d652d8d65d4e554a, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:30,996 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:30,996 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing d8089a176acb4ff8d652d8d65d4e554a, disabling compactions & flushes 2023-10-04T08:12:30,996 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:30,997 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:30,997 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. after waiting 0 ms 2023-10-04T08:12:30,997 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:30,997 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:30,997 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for d8089a176acb4ff8d652d8d65d4e554a: 2023-10-04T08:12:30,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:30,998 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1696407150998"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407150998"}]},"ts":"1696407150998"} 2023-10-04T08:12:31,001 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:31,002 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:31,002 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407151002"}]},"ts":"1696407151002"} 2023-10-04T08:12:31,004 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:31,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=d8089a176acb4ff8d652d8d65d4e554a, ASSIGN}] 2023-10-04T08:12:31,010 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=d8089a176acb4ff8d652d8d65d4e554a, ASSIGN 2023-10-04T08:12:31,011 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=d8089a176acb4ff8d652d8d65d4e554a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:31,023 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:31,138 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:31,162 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=d8089a176acb4ff8d652d8d65d4e554a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:31,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure d8089a176acb4ff8d652d8d65d4e554a, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:31,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2023-10-04T08:12:31,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:31,172 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2023-10-04T08:12:31,324 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:31,324 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7254): Opening region: {ENCODED => d8089a176acb4ff8d652d8d65d4e554a, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:31,324 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:31,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7296): checking encryption for d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,325 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7299): checking classloading for d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,326 INFO [StoreOpener-d8089a176acb4ff8d652d8d65d4e554a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,328 INFO [StoreOpener-d8089a176acb4ff8d652d8d65d4e554a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8089a176acb4ff8d652d8d65d4e554a columnFamilyName testFamily 2023-10-04T08:12:31,328 DEBUG [StoreOpener-d8089a176acb4ff8d652d8d65d4e554a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:31,329 INFO [StoreOpener-d8089a176acb4ff8d652d8d65d4e554a-1 {}] regionserver.HStore(324): Store=d8089a176acb4ff8d652d8d65d4e554a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:31,330 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,330 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,333 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1076): writing seq id for d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:12:31,336 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:31,336 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1093): Opened d8089a176acb4ff8d652d8d65d4e554a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62723963, jitterRate=-0.06534011662006378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:31,337 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(992): Region open journal for d8089a176acb4ff8d652d8d65d4e554a: 2023-10-04T08:12:31,338 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a., pid=32, masterSystemTime=1696407151319 2023-10-04T08:12:31,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:31,340 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:12:31,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=d8089a176acb4ff8d652d8d65d4e554a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:31,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-10-04T08:12:31,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure d8089a176acb4ff8d652d8d65d4e554a, server=jenkins-hbase3.apache.org,45997,1696407041329 in 177 msec 2023-10-04T08:12:31,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-04T08:12:31,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=d8089a176acb4ff8d652d8d65d4e554a, ASSIGN in 337 msec 2023-10-04T08:12:31,348 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:31,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407151348"}]},"ts":"1696407151348"} 2023-10-04T08:12:31,350 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:31,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:31,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 784 msec 2023-10-04T08:12:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=30 2023-10-04T08:12:31,576 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2023-10-04T08:12:31,576 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:31,577 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:31,581 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:31,581 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:31,581 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2023-10-04T08:12:31,583 DEBUG [hconnection-0x64af218a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:31,586 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44638, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:31,595 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:31,598 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:31,600 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885176 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:12:31,600 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2023-10-04T08:12:36,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:37,172 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2023-10-04T08:12:38,101 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2023-10-04T08:12:38,101 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e12bbc1 to 127.0.0.1:50391 2023-10-04T08:12:38,102 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:38,102 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:38,102 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:38,102 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:38,102 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c47ca36 to 127.0.0.1:50391 2023-10-04T08:12:38,114 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=210 (was 209) - Thread LEAK? -, OpenFileDescriptor=487 (was 487), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=115 (was 119), ProcessCount=171 (was 172), AvailableMemoryMB=6316 (was 6318) 2023-10-04T08:12:38,124 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=210, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=115, ProcessCount=171, AvailableMemoryMB=6316 2023-10-04T08:12:38,127 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x702ae4f9 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:38,133 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x702ae4f9 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6012c3b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:38,134 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x04258de5 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:38,138 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729c4822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:38,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2023-10-04T08:12:38,144 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:38,144 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:38,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2023-10-04T08:12:38,145 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-10-04T08:12:38,559 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => aa20ab650b256298cc3fd6d02def3716, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:38,970 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:38,970 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing aa20ab650b256298cc3fd6d02def3716, disabling compactions & flushes 2023-10-04T08:12:38,970 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:38,970 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:38,971 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. after waiting 0 ms 2023-10-04T08:12:38,971 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:38,971 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:38,971 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for aa20ab650b256298cc3fd6d02def3716: 2023-10-04T08:12:38,972 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:38,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1696407158972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407158972"}]},"ts":"1696407158972"} 2023-10-04T08:12:38,975 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:38,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:38,976 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407158976"}]},"ts":"1696407158976"} 2023-10-04T08:12:38,978 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:38,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=aa20ab650b256298cc3fd6d02def3716, ASSIGN}] 2023-10-04T08:12:38,984 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=aa20ab650b256298cc3fd6d02def3716, ASSIGN 2023-10-04T08:12:38,985 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=aa20ab650b256298cc3fd6d02def3716, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:39,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=aa20ab650b256298cc3fd6d02def3716, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:39,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure aa20ab650b256298cc3fd6d02def3716, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-10-04T08:12:39,296 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:39,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7254): Opening region: {ENCODED => aa20ab650b256298cc3fd6d02def3716, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:39,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:39,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7296): checking encryption for aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7299): checking classloading for aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,298 INFO [StoreOpener-aa20ab650b256298cc3fd6d02def3716-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,300 INFO [StoreOpener-aa20ab650b256298cc3fd6d02def3716-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa20ab650b256298cc3fd6d02def3716 columnFamilyName testFamily 2023-10-04T08:12:39,300 DEBUG [StoreOpener-aa20ab650b256298cc3fd6d02def3716-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:39,301 INFO [StoreOpener-aa20ab650b256298cc3fd6d02def3716-1 {}] regionserver.HStore(324): Store=aa20ab650b256298cc3fd6d02def3716/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:39,302 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,306 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1076): writing seq id for aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:12:39,308 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:39,309 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): Opened aa20ab650b256298cc3fd6d02def3716; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63129173, jitterRate=-0.05930201709270477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:39,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(992): Region open journal for aa20ab650b256298cc3fd6d02def3716: 2023-10-04T08:12:39,310 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716., pid=35, masterSystemTime=1696407159291 2023-10-04T08:12:39,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:39,313 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:12:39,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=aa20ab650b256298cc3fd6d02def3716, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:39,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-04T08:12:39,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure aa20ab650b256298cc3fd6d02def3716, server=jenkins-hbase3.apache.org,45997,1696407041329 in 177 msec 2023-10-04T08:12:39,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=34, resume processing ppid=33 2023-10-04T08:12:39,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=aa20ab650b256298cc3fd6d02def3716, ASSIGN in 335 msec 2023-10-04T08:12:39,320 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:39,320 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407159320"}]},"ts":"1696407159320"} 2023-10-04T08:12:39,322 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:39,326 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:39,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.1850 sec 2023-10-04T08:12:41,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2023-10-04T08:12:41,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:41,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2023-10-04T08:12:41,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=33 2023-10-04T08:12:41,150 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2023-10-04T08:12:41,150 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:41,150 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:41,155 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:41,156 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:41,156 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2023-10-04T08:12:41,158 DEBUG [hconnection-0x7b2f523e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:41,161 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46452, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:41,171 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:41,175 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:41,178 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885180 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:12:41,178 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2023-10-04T08:12:42,679 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1504 2023-10-04T08:12:42,680 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x04258de5 to 127.0.0.1:50391 2023-10-04T08:12:42,680 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:42,680 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:42,680 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:42,680 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:42,680 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x702ae4f9 to 127.0.0.1:50391 2023-10-04T08:12:42,693 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=213 (was 210) - Thread LEAK? -, OpenFileDescriptor=489 (was 487) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=115 (was 115), ProcessCount=171 (was 171), AvailableMemoryMB=6315 (was 6316) 2023-10-04T08:12:42,704 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=213, OpenFileDescriptor=489, MaxFileDescriptor=60000, SystemLoadAverage=115, ProcessCount=171, AvailableMemoryMB=6315 2023-10-04T08:12:42,705 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6e06fbfe to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:42,710 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x6e06fbfe {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221b92cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:42,711 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5bc61c07 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:42,715 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425f6637, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:42,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2023-10-04T08:12:42,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:42,721 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:42,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2023-10-04T08:12:42,722 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-04T08:12:43,136 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 87fdd579d0b26040c8abef46e9d1ae20, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 87fdd579d0b26040c8abef46e9d1ae20, disabling compactions & flushes 2023-10-04T08:12:43,146 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. after waiting 0 ms 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,146 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,146 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 87fdd579d0b26040c8abef46e9d1ae20: 2023-10-04T08:12:43,148 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:43,148 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1696407163148"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407163148"}]},"ts":"1696407163148"} 2023-10-04T08:12:43,151 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:43,152 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:43,152 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407163152"}]},"ts":"1696407163152"} 2023-10-04T08:12:43,154 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:43,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=87fdd579d0b26040c8abef46e9d1ae20, ASSIGN}] 2023-10-04T08:12:43,160 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=87fdd579d0b26040c8abef46e9d1ae20, ASSIGN 2023-10-04T08:12:43,161 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=87fdd579d0b26040c8abef46e9d1ae20, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:43,173 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2023-10-04T08:12:43,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=87fdd579d0b26040c8abef46e9d1ae20, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:43,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 87fdd579d0b26040c8abef46e9d1ae20, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:43,472 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,472 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 87fdd579d0b26040c8abef46e9d1ae20, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:43,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:43,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,475 INFO [StoreOpener-87fdd579d0b26040c8abef46e9d1ae20-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,476 INFO [StoreOpener-87fdd579d0b26040c8abef46e9d1ae20-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87fdd579d0b26040c8abef46e9d1ae20 columnFamilyName testFamily 2023-10-04T08:12:43,477 DEBUG [StoreOpener-87fdd579d0b26040c8abef46e9d1ae20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:43,477 INFO [StoreOpener-87fdd579d0b26040c8abef46e9d1ae20-1 {}] regionserver.HStore(324): Store=87fdd579d0b26040c8abef46e9d1ae20/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:43,478 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:12:43,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:43,486 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 87fdd579d0b26040c8abef46e9d1ae20; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65569934, jitterRate=-0.022931843996047974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:43,486 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 87fdd579d0b26040c8abef46e9d1ae20: 2023-10-04T08:12:43,488 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20., pid=38, masterSystemTime=1696407163466 2023-10-04T08:12:43,490 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,490 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:12:43,491 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=87fdd579d0b26040c8abef46e9d1ae20, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:43,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-10-04T08:12:43,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 87fdd579d0b26040c8abef46e9d1ae20, server=jenkins-hbase3.apache.org,45997,1696407041329 in 178 msec 2023-10-04T08:12:43,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-04T08:12:43,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=87fdd579d0b26040c8abef46e9d1ae20, ASSIGN in 336 msec 2023-10-04T08:12:43,497 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:43,497 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407163497"}]},"ts":"1696407163497"} 2023-10-04T08:12:43,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:43,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:43,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 783 msec 2023-10-04T08:12:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-04T08:12:43,725 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2023-10-04T08:12:43,725 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:43,726 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:43,731 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:43,732 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:43,732 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2023-10-04T08:12:43,734 DEBUG [hconnection-0x58ec8330-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:43,737 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46458, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:43,747 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:43,751 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x71104925 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:43,756 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147d9fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:43,756 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:43,759 DEBUG [hconnection-0x549c0841-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:43,764 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46462, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:43,768 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:12:43,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885185 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:43,770 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:12:43,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885185 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:12:43,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-10-04T08:12:44,780 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-10-04T08:12:44.779Z, RpcRetryingCaller{globalStartTime=2023-10-04T08:12:43.770Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-10-04T08:12:44.779Z, RpcRetryingCaller{globalStartTime=2023-10-04T08:12:43.770Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:12:44,780 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1010 2023-10-04T08:12:44,781 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x71104925 to 127.0.0.1:50391 2023-10-04T08:12:44,781 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:44,781 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5bc61c07 to 127.0.0.1:50391 2023-10-04T08:12:44,781 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:44,781 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:44,781 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:44,782 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:44,782 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6e06fbfe to 127.0.0.1:50391 2023-10-04T08:12:44,795 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=217 (was 213) - Thread LEAK? -, OpenFileDescriptor=496 (was 489) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=131 (was 115) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=6313 (was 6315) 2023-10-04T08:12:44,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:44,808 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=217, OpenFileDescriptor=496, MaxFileDescriptor=60000, SystemLoadAverage=131, ProcessCount=171, AvailableMemoryMB=6311 2023-10-04T08:12:44,810 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x66b721db to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:44,815 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x66b721db {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ede0ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:44,816 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41049690 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:44,821 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4361ee93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:44,824 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:44,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2023-10-04T08:12:44,827 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:44,827 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:44,827 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2023-10-04T08:12:44,829 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-04T08:12:45,257 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2b8b1fed447fcf6ffb03157a004d4e73, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:45,271 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:12:45,312 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2023-10-04T08:12:45,318 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 015af7fd5d2067013019f6874c98f002 changed from -1.0 to 0.0, refreshing cache 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 2b8b1fed447fcf6ffb03157a004d4e73, disabling compactions & flushes 2023-10-04T08:12:45,667 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. after waiting 0 ms 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:45,667 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:45,667 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2b8b1fed447fcf6ffb03157a004d4e73: 2023-10-04T08:12:45,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:45,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1696407165669"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407165669"}]},"ts":"1696407165669"} 2023-10-04T08:12:45,671 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:45,672 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:45,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407165673"}]},"ts":"1696407165673"} 2023-10-04T08:12:45,674 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:45,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2b8b1fed447fcf6ffb03157a004d4e73, ASSIGN}] 2023-10-04T08:12:45,680 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2b8b1fed447fcf6ffb03157a004d4e73, ASSIGN 2023-10-04T08:12:45,681 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2b8b1fed447fcf6ffb03157a004d4e73, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:45,832 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=2b8b1fed447fcf6ffb03157a004d4e73, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-04T08:12:45,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 2b8b1fed447fcf6ffb03157a004d4e73, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:45,991 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:45,991 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => 2b8b1fed447fcf6ffb03157a004d4e73, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:45,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:45,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:45,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:45,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:45,994 INFO [StoreOpener-2b8b1fed447fcf6ffb03157a004d4e73-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:45,995 INFO [StoreOpener-2b8b1fed447fcf6ffb03157a004d4e73-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b8b1fed447fcf6ffb03157a004d4e73 columnFamilyName testFamily 2023-10-04T08:12:45,996 DEBUG [StoreOpener-2b8b1fed447fcf6ffb03157a004d4e73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:45,996 INFO [StoreOpener-2b8b1fed447fcf6ffb03157a004d4e73-1 {}] regionserver.HStore(324): Store=2b8b1fed447fcf6ffb03157a004d4e73/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:45,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:45,998 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:46,001 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:12:46,005 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:46,005 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1093): Opened 2b8b1fed447fcf6ffb03157a004d4e73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72780147, jitterRate=0.08450870215892792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:46,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(992): Region open journal for 2b8b1fed447fcf6ffb03157a004d4e73: 2023-10-04T08:12:46,007 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73., pid=41, masterSystemTime=1696407165986 2023-10-04T08:12:46,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:46,009 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:12:46,010 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=2b8b1fed447fcf6ffb03157a004d4e73, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:46,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-04T08:12:46,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 2b8b1fed447fcf6ffb03157a004d4e73, server=jenkins-hbase3.apache.org,45997,1696407041329 in 177 msec 2023-10-04T08:12:46,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-04T08:12:46,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2b8b1fed447fcf6ffb03157a004d4e73, ASSIGN in 335 msec 2023-10-04T08:12:46,016 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:46,016 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407166016"}]},"ts":"1696407166016"} 2023-10-04T08:12:46,018 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:46,022 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:46,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.1980 sec 2023-10-04T08:12:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-04T08:12:47,835 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2023-10-04T08:12:47,835 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:47,835 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:47,839 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:47,839 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:47,839 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2023-10-04T08:12:47,841 DEBUG [hconnection-0x6882f58c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:47,843 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46466, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:47,852 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:47,854 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:12:47,856 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:47,857 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-10-04T08:12:49,174 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2023-10-04T08:12:49,175 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2023-10-04T08:12:51,494 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:12:51,541 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2023-10-04T08:12:51,541 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:51,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2023-10-04T08:12:51,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:12:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 12 bytes 2023-10-04T08:12:54,867 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:54,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.31.12.81:46466 deadline: 1696407176856, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:12:54,868 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:12:54,870 DEBUG [Listener at localhost/43533 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885189 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:12:54,870 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7015 2023-10-04T08:12:54,870 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41049690 to 127.0.0.1:50391 2023-10-04T08:12:54,870 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:54,871 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:12:54,871 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:12:54,871 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:12:54,871 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x66b721db to 127.0.0.1:50391 2023-10-04T08:12:54,882 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=213 (was 217), OpenFileDescriptor=485 (was 496), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=126 (was 131), ProcessCount=170 (was 171), AvailableMemoryMB=6293 (was 6311) 2023-10-04T08:12:54,894 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=213, OpenFileDescriptor=485, MaxFileDescriptor=60000, SystemLoadAverage=126, ProcessCount=170, AvailableMemoryMB=6293 2023-10-04T08:12:54,896 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x72e1fa16 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:54,900 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x72e1fa16 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a2a1e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:54,901 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2fdca11d to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:12:54,905 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffbbfc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:12:54,908 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:12:54,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-10-04T08:12:54,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:12:54,910 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:54,911 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2023-10-04T08:12:54,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:12:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-04T08:12:55,325 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 71b5c7d5f3c2000b3e5dbfd8bb238abb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 71b5c7d5f3c2000b3e5dbfd8bb238abb, disabling compactions & flushes 2023-10-04T08:12:55,736 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. after waiting 0 ms 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:55,736 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:55,736 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 71b5c7d5f3c2000b3e5dbfd8bb238abb: 2023-10-04T08:12:55,737 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:12:55,738 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1696407175737"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407175737"}]},"ts":"1696407175737"} 2023-10-04T08:12:55,740 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:12:55,741 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:12:55,741 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407175741"}]},"ts":"1696407175741"} 2023-10-04T08:12:55,743 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:12:55,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=71b5c7d5f3c2000b3e5dbfd8bb238abb, ASSIGN}] 2023-10-04T08:12:55,749 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=71b5c7d5f3c2000b3e5dbfd8bb238abb, ASSIGN 2023-10-04T08:12:55,749 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=71b5c7d5f3c2000b3e5dbfd8bb238abb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:12:55,900 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=71b5c7d5f3c2000b3e5dbfd8bb238abb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:55,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 71b5c7d5f3c2000b3e5dbfd8bb238abb, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:12:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-04T08:12:56,062 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:56,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7254): Opening region: {ENCODED => 71b5c7d5f3c2000b3e5dbfd8bb238abb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:12:56,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:12:56,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7296): checking encryption for 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7299): checking classloading for 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,064 INFO [StoreOpener-71b5c7d5f3c2000b3e5dbfd8bb238abb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,066 INFO [StoreOpener-71b5c7d5f3c2000b3e5dbfd8bb238abb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71b5c7d5f3c2000b3e5dbfd8bb238abb columnFamilyName testFamily 2023-10-04T08:12:56,066 DEBUG [StoreOpener-71b5c7d5f3c2000b3e5dbfd8bb238abb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:12:56,067 INFO [StoreOpener-71b5c7d5f3c2000b3e5dbfd8bb238abb-1 {}] regionserver.HStore(324): Store=71b5c7d5f3c2000b3e5dbfd8bb238abb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:12:56,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1076): writing seq id for 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:12:56,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:12:56,075 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1093): Opened 71b5c7d5f3c2000b3e5dbfd8bb238abb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65250382, jitterRate=-0.02769353985786438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:12:56,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(992): Region open journal for 71b5c7d5f3c2000b3e5dbfd8bb238abb: 2023-10-04T08:12:56,077 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., pid=44, masterSystemTime=1696407176055 2023-10-04T08:12:56,079 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:56,079 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:12:56,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=71b5c7d5f3c2000b3e5dbfd8bb238abb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:12:56,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-10-04T08:12:56,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 71b5c7d5f3c2000b3e5dbfd8bb238abb, server=jenkins-hbase3.apache.org,45997,1696407041329 in 178 msec 2023-10-04T08:12:56,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-04T08:12:56,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=71b5c7d5f3c2000b3e5dbfd8bb238abb, ASSIGN in 337 msec 2023-10-04T08:12:56,087 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:12:56,087 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407176087"}]},"ts":"1696407176087"} 2023-10-04T08:12:56,089 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:12:56,092 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:12:56,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1840 sec 2023-10-04T08:12:57,637 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666476 expired clientIPAndPort=172.31.12.81:51450, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:12:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-04T08:12:57,916 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2023-10-04T08:12:57,916 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:12:57,916 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:57,920 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2023-10-04T08:12:57,920 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:12:57,920 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2023-10-04T08:12:57,922 DEBUG [hconnection-0x41d4929f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:57,925 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36380, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:57,934 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:12:57,937 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:12:57,938 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:12:57,939 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x72e1fa16 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:12:57,939 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x72e1fa16 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:12:57,940 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36392, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:12:57,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:12:57,944 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:13:01,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-10-04T08:13:01,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:01,565 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:01,598 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2023-10-04T08:13:03,950 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2023-10-04T08:13:03,951 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2023-10-04T08:13:03,951 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:13:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 89 bytes 2023-10-04T08:13:04,643 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666474 expired clientIPAndPort=172.31.12.81:51450, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:13:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:13:04,964 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:13:06,939 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1976ms, rpcTimeout=1974ms 2023-10-04T08:13:06,939 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1976ms, rpcTimeout=1974ms 2023-10-04T08:13:06,939 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:13:06,940 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-10-04T08:13:03.951Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6006ms, rpcTimeout=6000ms 2023-10-04T08:13:06.939Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1976ms, rpcTimeout=1974ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1976ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1976ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-10-04T08:13:06,942 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9005 2023-10-04T08:13:06,943 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2fdca11d to 127.0.0.1:50391 2023-10-04T08:13:06,943 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:06,943 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:06,943 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:06,943 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:06,943 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x72e1fa16 to 127.0.0.1:50391 2023-10-04T08:13:06,954 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=211 (was 213), OpenFileDescriptor=478 (was 485), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=122 (was 126), ProcessCount=170 (was 170), AvailableMemoryMB=6281 (was 6293) 2023-10-04T08:13:06,965 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=211, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=122, ProcessCount=170, AvailableMemoryMB=6280 2023-10-04T08:13:06,967 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x122d3e86 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:06,971 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x122d3e86 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae1671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:06,972 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x76b3e8a0 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:06,976 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b72a7b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:06,979 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:06,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2023-10-04T08:13:06,981 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:06,981 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:06,981 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2023-10-04T08:13:06,982 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-04T08:13:07,397 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => daa135c3e71fa3e0d3b0ca3000158a02, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:07,807 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:07,808 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing daa135c3e71fa3e0d3b0ca3000158a02, disabling compactions & flushes 2023-10-04T08:13:07,808 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:07,808 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:07,808 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. after waiting 0 ms 2023-10-04T08:13:07,808 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:07,808 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:07,808 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for daa135c3e71fa3e0d3b0ca3000158a02: 2023-10-04T08:13:07,809 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:07,809 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1696407187809"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407187809"}]},"ts":"1696407187809"} 2023-10-04T08:13:07,812 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:07,813 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:07,813 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407187813"}]},"ts":"1696407187813"} 2023-10-04T08:13:07,815 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:13:07,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=daa135c3e71fa3e0d3b0ca3000158a02, ASSIGN}] 2023-10-04T08:13:07,822 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=daa135c3e71fa3e0d3b0ca3000158a02, ASSIGN 2023-10-04T08:13:07,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=daa135c3e71fa3e0d3b0ca3000158a02, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:07,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=daa135c3e71fa3e0d3b0ca3000158a02, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:07,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure daa135c3e71fa3e0d3b0ca3000158a02, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-04T08:13:08,135 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:08,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7254): Opening region: {ENCODED => daa135c3e71fa3e0d3b0ca3000158a02, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:08,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,136 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:08,136 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7296): checking encryption for daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,136 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7299): checking classloading for daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,137 INFO [StoreOpener-daa135c3e71fa3e0d3b0ca3000158a02-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,139 INFO [StoreOpener-daa135c3e71fa3e0d3b0ca3000158a02-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daa135c3e71fa3e0d3b0ca3000158a02 columnFamilyName testFamily 2023-10-04T08:13:08,139 DEBUG [StoreOpener-daa135c3e71fa3e0d3b0ca3000158a02-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:08,139 INFO [StoreOpener-daa135c3e71fa3e0d3b0ca3000158a02-1 {}] regionserver.HStore(324): Store=daa135c3e71fa3e0d3b0ca3000158a02/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:08,140 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,144 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1076): writing seq id for daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:13:08,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:08,147 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1093): Opened daa135c3e71fa3e0d3b0ca3000158a02; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68069524, jitterRate=0.01431494951248169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:08,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(992): Region open journal for daa135c3e71fa3e0d3b0ca3000158a02: 2023-10-04T08:13:08,149 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02., pid=47, masterSystemTime=1696407188128 2023-10-04T08:13:08,151 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:08,151 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:13:08,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=daa135c3e71fa3e0d3b0ca3000158a02, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:08,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-10-04T08:13:08,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure daa135c3e71fa3e0d3b0ca3000158a02, server=jenkins-hbase3.apache.org,45997,1696407041329 in 178 msec 2023-10-04T08:13:08,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-10-04T08:13:08,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=daa135c3e71fa3e0d3b0ca3000158a02, ASSIGN in 335 msec 2023-10-04T08:13:08,158 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:08,158 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407188158"}]},"ts":"1696407188158"} 2023-10-04T08:13:08,159 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:13:08,162 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:08,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.1840 sec 2023-10-04T08:13:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=45 2023-10-04T08:13:09,989 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2023-10-04T08:13:09,989 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:13:09,989 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:09,993 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2023-10-04T08:13:09,993 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:09,993 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2023-10-04T08:13:09,995 DEBUG [hconnection-0x4860afdd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:09,997 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54372, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:10,006 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:10,010 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1622b6f5 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:10,015 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c95c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:10,015 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:13:10,017 DEBUG [hconnection-0x78a73170-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:10,018 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:10,022 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:13:10,022 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885202 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:10,023 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:13:10,024 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885202 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:10,024 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-10-04T08:13:10,748 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666469 expired clientIPAndPort=172.31.12.81:55446, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:13:11,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2023-10-04T08:13:11,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:11,464 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:13:13,350 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666465 expired clientIPAndPort=172.31.12.81:46226, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:13:13,350 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666463 expired clientIPAndPort=172.31.12.81:46226, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:13:13,350 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666462 expired clientIPAndPort=172.31.12.81:46226, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:13:13,672 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:13,708 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2023-10-04T08:13:16,031 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-10-04T08:13:16.030Z, RpcRetryingCaller{globalStartTime=2023-10-04T08:13:10.023Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2023-10-04T08:13:16.030Z, RpcRetryingCaller{globalStartTime=2023-10-04T08:13:10.023Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:13:16,031 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6007 2023-10-04T08:13:16,031 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1622b6f5 to 127.0.0.1:50391 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x76b3e8a0 to 127.0.0.1:50391 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:16,032 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:16,032 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x122d3e86 to 127.0.0.1:50391 2023-10-04T08:13:16,045 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=217 (was 211) - Thread LEAK? -, OpenFileDescriptor=486 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=119 (was 122), ProcessCount=170 (was 170), AvailableMemoryMB=6252 (was 6280) 2023-10-04T08:13:16,056 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=217, OpenFileDescriptor=486, MaxFileDescriptor=60000, SystemLoadAverage=119, ProcessCount=170, AvailableMemoryMB=6252 2023-10-04T08:13:16,058 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3fd9aaf5 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:16,063 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x3fd9aaf5 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485e7ac8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:16,064 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x33679184 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:16,069 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3b4713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:16,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2023-10-04T08:13:16,074 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:16,074 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:16,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2023-10-04T08:13:16,075 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-04T08:13:16,487 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fdbb6b7904791beef7b9e29a91436269, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:16,525 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing fdbb6b7904791beef7b9e29a91436269, disabling compactions & flushes 2023-10-04T08:13:16,898 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. after waiting 0 ms 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:16,898 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:16,898 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for fdbb6b7904791beef7b9e29a91436269: 2023-10-04T08:13:16,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:16,899 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1696407196899"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407196899"}]},"ts":"1696407196899"} 2023-10-04T08:13:16,901 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:16,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:16,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407196902"}]},"ts":"1696407196902"} 2023-10-04T08:13:16,903 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2023-10-04T08:13:16,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=fdbb6b7904791beef7b9e29a91436269, ASSIGN}] 2023-10-04T08:13:16,909 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=fdbb6b7904791beef7b9e29a91436269, ASSIGN 2023-10-04T08:13:16,910 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=fdbb6b7904791beef7b9e29a91436269, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:17,061 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=fdbb6b7904791beef7b9e29a91436269, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:17,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure fdbb6b7904791beef7b9e29a91436269, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-04T08:13:17,220 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:17,220 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7254): Opening region: {ENCODED => fdbb6b7904791beef7b9e29a91436269, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:17,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:17,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7296): checking encryption for fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,221 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7299): checking classloading for fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,222 INFO [StoreOpener-fdbb6b7904791beef7b9e29a91436269-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,224 INFO [StoreOpener-fdbb6b7904791beef7b9e29a91436269-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fdbb6b7904791beef7b9e29a91436269 columnFamilyName testFamily 2023-10-04T08:13:17,224 DEBUG [StoreOpener-fdbb6b7904791beef7b9e29a91436269-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:17,225 INFO [StoreOpener-fdbb6b7904791beef7b9e29a91436269-1 {}] regionserver.HStore(324): Store=fdbb6b7904791beef7b9e29a91436269/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:17,226 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,226 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,229 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1076): writing seq id for fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:13:17,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:17,232 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): Opened fdbb6b7904791beef7b9e29a91436269; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62499088, jitterRate=-0.06869101524353027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:17,233 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(992): Region open journal for fdbb6b7904791beef7b9e29a91436269: 2023-10-04T08:13:17,234 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., pid=50, masterSystemTime=1696407197215 2023-10-04T08:13:17,236 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:17,236 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:13:17,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=fdbb6b7904791beef7b9e29a91436269, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:17,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-10-04T08:13:17,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure fdbb6b7904791beef7b9e29a91436269, server=jenkins-hbase3.apache.org,45997,1696407041329 in 175 msec 2023-10-04T08:13:17,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-10-04T08:13:17,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=fdbb6b7904791beef7b9e29a91436269, ASSIGN in 332 msec 2023-10-04T08:13:17,242 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:17,243 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407197243"}]},"ts":"1696407197243"} 2023-10-04T08:13:17,244 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2023-10-04T08:13:17,247 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:17,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1750 sec 2023-10-04T08:13:18,454 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666456 expired clientIPAndPort=172.31.12.81:46242, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:13:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-04T08:13:19,079 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2023-10-04T08:13:19,079 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2023-10-04T08:13:19,080 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:19,083 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2023-10-04T08:13:19,083 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:19,084 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2023-10-04T08:13:19,086 DEBUG [hconnection-0xf72d9b2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:19,087 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46958, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:19,096 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:19,099 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:13:19,100 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:13:19,101 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x3fd9aaf5 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:13:19,102 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x3fd9aaf5 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:19,103 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46962, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:19,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:13:19,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:13:19,709 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2023-10-04T08:13:20,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2023-10-04T08:13:20,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2023-10-04T08:13:20,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:13:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 89 bytes 2023-10-04T08:13:21,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2023-10-04T08:13:21,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:13:21,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:13:22,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=987ms, rpcTimeout=977ms 2023-10-04T08:13:22,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=987ms, rpcTimeout=977ms 2023-10-04T08:13:22,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:13:22,110 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-10-04T08:13:20.110Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2023-10-04T08:13:22.109Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=987ms, rpcTimeout=977ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=987ms, rpcTimeout=977ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=987ms, rpcTimeout=977ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-10-04T08:13:22,114 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3014 2023-10-04T08:13:22,114 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x33679184 to 127.0.0.1:50391 2023-10-04T08:13:22,114 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:22,114 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:22,114 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:22,114 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:22,114 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3fd9aaf5 to 127.0.0.1:50391 2023-10-04T08:13:22,126 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=214 (was 217), OpenFileDescriptor=482 (was 486), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=117 (was 119), ProcessCount=170 (was 170), AvailableMemoryMB=6250 (was 6252) 2023-10-04T08:13:22,137 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=117, ProcessCount=170, AvailableMemoryMB=6249 2023-10-04T08:13:22,138 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x54042f3b to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:22,143 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x54042f3b {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64600486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:22,144 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x29557fc3 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:22,153 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de502a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:22,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2023-10-04T08:13:22,158 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:22,158 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:22,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2023-10-04T08:13:22,159 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-04T08:13:22,571 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7846ddcf7f67173b45f510faad9a2446, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:22,623 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:13:22,722 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:22,963 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666451 expired clientIPAndPort=172.31.12.81:50586, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 7846ddcf7f67173b45f510faad9a2446, disabling compactions & flushes 2023-10-04T08:13:22,981 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. after waiting 0 ms 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:22,981 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:22,981 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7846ddcf7f67173b45f510faad9a2446: 2023-10-04T08:13:22,982 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:22,983 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1696407202983"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407202983"}]},"ts":"1696407202983"} 2023-10-04T08:13:22,985 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:22,986 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:22,986 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407202986"}]},"ts":"1696407202986"} 2023-10-04T08:13:22,988 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:22,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7846ddcf7f67173b45f510faad9a2446, ASSIGN}] 2023-10-04T08:13:22,994 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7846ddcf7f67173b45f510faad9a2446, ASSIGN 2023-10-04T08:13:22,995 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7846ddcf7f67173b45f510faad9a2446, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:23,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=7846ddcf7f67173b45f510faad9a2446, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:23,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 7846ddcf7f67173b45f510faad9a2446, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-04T08:13:23,305 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:23,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 7846ddcf7f67173b45f510faad9a2446, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:23,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:23,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,307 INFO [StoreOpener-7846ddcf7f67173b45f510faad9a2446-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,308 INFO [StoreOpener-7846ddcf7f67173b45f510faad9a2446-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7846ddcf7f67173b45f510faad9a2446 columnFamilyName testFamily 2023-10-04T08:13:23,308 DEBUG [StoreOpener-7846ddcf7f67173b45f510faad9a2446-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:23,309 INFO [StoreOpener-7846ddcf7f67173b45f510faad9a2446-1 {}] regionserver.HStore(324): Store=7846ddcf7f67173b45f510faad9a2446/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:23,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:13:23,315 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:23,316 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 7846ddcf7f67173b45f510faad9a2446; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66030631, jitterRate=-0.01606692373752594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:23,316 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 7846ddcf7f67173b45f510faad9a2446: 2023-10-04T08:13:23,317 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446., pid=53, masterSystemTime=1696407203300 2023-10-04T08:13:23,319 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:23,319 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:13:23,319 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=7846ddcf7f67173b45f510faad9a2446, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:23,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-04T08:13:23,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 7846ddcf7f67173b45f510faad9a2446, server=jenkins-hbase3.apache.org,45997,1696407041329 in 173 msec 2023-10-04T08:13:23,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-10-04T08:13:23,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=7846ddcf7f67173b45f510faad9a2446, ASSIGN in 331 msec 2023-10-04T08:13:23,325 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:23,325 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407203325"}]},"ts":"1696407203325"} 2023-10-04T08:13:23,327 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:23,330 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:23,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1740 sec 2023-10-04T08:13:24,964 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666449 expired clientIPAndPort=172.31.12.81:50586, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:13:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-04T08:13:25,164 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2023-10-04T08:13:25,164 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:25,164 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:25,167 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:25,167 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:25,167 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2023-10-04T08:13:25,170 DEBUG [hconnection-0xb366ba2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:25,172 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44438, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:25,179 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:25,181 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-10-04T08:13:25,182 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:13:25,183 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x54042f3b {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:13:25,183 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x54042f3b {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:25,184 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44454, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:13:25,187 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-10-04T08:13:25,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885214 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,188 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-10-04T08:13:25,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885214 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666402 2023-10-04T08:13:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:44454 deadline: 1696407208187, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:25,189 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885214 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:13:25,190 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-10-04T08:13:25,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885215 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,191 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-10-04T08:13:25,191 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:25,192 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:25,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885216 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666400 2023-10-04T08:13:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:44454 deadline: 1696407208191, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:25,193 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885216 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:13:25,194 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:25,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885217 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-452929323842666399 2023-10-04T08:13:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:44454 deadline: 1696407208193, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:25,195 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885217 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:13:25,196 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:25,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885218 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:25,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-452929323842666398 2023-10-04T08:13:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:44454 deadline: 1696407208196, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:25,199 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885218 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:13:25,200 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:25,200 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x29557fc3 to 127.0.0.1:50391 2023-10-04T08:13:25,200 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:25,200 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:25,200 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:25,201 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:25,201 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x54042f3b to 127.0.0.1:50391 2023-10-04T08:13:25,212 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=216 (was 214) - Thread LEAK? -, OpenFileDescriptor=484 (was 482) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=148 (was 117) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=6174 (was 6249) 2023-10-04T08:13:25,222 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=216, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=148, ProcessCount=170, AvailableMemoryMB=6174 2023-10-04T08:13:25,224 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5589e7db to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:25,228 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x5589e7db {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152ead1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:25,230 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0a7d634b to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:25,235 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a54a1b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:25,237 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:25,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2023-10-04T08:13:25,239 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:25,239 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:25,240 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2023-10-04T08:13:25,240 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-04T08:13:25,657 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3443a3e340fdb17279ac1f79e8b74717, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:25,710 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 3443a3e340fdb17279ac1f79e8b74717, disabling compactions & flushes 2023-10-04T08:13:26,068 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. after waiting 0 ms 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,068 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,068 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3443a3e340fdb17279ac1f79e8b74717: 2023-10-04T08:13:26,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:26,070 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1696407206070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407206070"}]},"ts":"1696407206070"} 2023-10-04T08:13:26,072 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:26,073 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:26,073 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407206073"}]},"ts":"1696407206073"} 2023-10-04T08:13:26,074 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:26,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=3443a3e340fdb17279ac1f79e8b74717, ASSIGN}] 2023-10-04T08:13:26,080 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=3443a3e340fdb17279ac1f79e8b74717, ASSIGN 2023-10-04T08:13:26,080 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=3443a3e340fdb17279ac1f79e8b74717, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:26,231 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=3443a3e340fdb17279ac1f79e8b74717, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:26,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 3443a3e340fdb17279ac1f79e8b74717, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-04T08:13:26,390 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7254): Opening region: {ENCODED => 3443a3e340fdb17279ac1f79e8b74717, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:26,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:26,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7296): checking encryption for 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7299): checking classloading for 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,392 INFO [StoreOpener-3443a3e340fdb17279ac1f79e8b74717-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,393 INFO [StoreOpener-3443a3e340fdb17279ac1f79e8b74717-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3443a3e340fdb17279ac1f79e8b74717 columnFamilyName testFamily 2023-10-04T08:13:26,393 DEBUG [StoreOpener-3443a3e340fdb17279ac1f79e8b74717-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:26,394 INFO [StoreOpener-3443a3e340fdb17279ac1f79e8b74717-1 {}] regionserver.HStore(324): Store=3443a3e340fdb17279ac1f79e8b74717/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:26,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,397 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1076): writing seq id for 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:13:26,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:26,400 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1093): Opened 3443a3e340fdb17279ac1f79e8b74717; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71312383, jitterRate=0.0626373142004013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:26,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(992): Region open journal for 3443a3e340fdb17279ac1f79e8b74717: 2023-10-04T08:13:26,401 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., pid=56, masterSystemTime=1696407206385 2023-10-04T08:13:26,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,403 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:13:26,403 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=3443a3e340fdb17279ac1f79e8b74717, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:26,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-10-04T08:13:26,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 3443a3e340fdb17279ac1f79e8b74717, server=jenkins-hbase3.apache.org,45997,1696407041329 in 172 msec 2023-10-04T08:13:26,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-10-04T08:13:26,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=3443a3e340fdb17279ac1f79e8b74717, ASSIGN in 329 msec 2023-10-04T08:13:26,409 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:26,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407206409"}]},"ts":"1696407206409"} 2023-10-04T08:13:26,411 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:26,415 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:26,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1780 sec 2023-10-04T08:13:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-04T08:13:28,244 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2023-10-04T08:13:28,244 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:28,245 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:28,248 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:28,248 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:28,248 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2023-10-04T08:13:28,250 DEBUG [hconnection-0x598592d3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:28,252 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:44470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:28,259 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:28,261 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:13:28,262 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:13:28,807 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:31,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2023-10-04T08:13:31,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:31,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2023-10-04T08:13:31,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:31,069 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666444 expired clientIPAndPort=172.31.12.81:50610, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:13:31,711 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2023-10-04T08:13:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 89 bytes 2023-10-04T08:13:35,282 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:13:37,271 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:13:37,271 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-10-04T08:13:37.271Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1987ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:13:37,272 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9010 2023-10-04T08:13:37,272 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0a7d634b to 127.0.0.1:50391 2023-10-04T08:13:37,272 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:37,273 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:37,273 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:37,273 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:37,273 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5589e7db to 127.0.0.1:50391 2023-10-04T08:13:37,284 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=212 (was 216), OpenFileDescriptor=478 (was 484), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=140 (was 148), ProcessCount=170 (was 170), AvailableMemoryMB=6181 (was 6174) - AvailableMemoryMB LEAK? - 2023-10-04T08:13:37,294 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=212, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=140, ProcessCount=170, AvailableMemoryMB=6181 2023-10-04T08:13:37,295 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x49154ac6 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:37,299 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x49154ac6 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e0df2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:37,300 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7fbf50c4 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:37,305 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4116591f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:37,307 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:37,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2023-10-04T08:13:37,309 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:37,309 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:37,309 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2023-10-04T08:13:37,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-04T08:13:37,728 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7cf4420553a66064d10b73e18ff7ce65, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 7cf4420553a66064d10b73e18ff7ce65, disabling compactions & flushes 2023-10-04T08:13:37,741 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. after waiting 0 ms 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:37,741 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:37,741 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7cf4420553a66064d10b73e18ff7ce65: 2023-10-04T08:13:37,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:37,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1696407217742"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407217742"}]},"ts":"1696407217742"} 2023-10-04T08:13:37,744 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:37,745 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:37,745 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407217745"}]},"ts":"1696407217745"} 2023-10-04T08:13:37,747 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:37,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=7cf4420553a66064d10b73e18ff7ce65, ASSIGN}] 2023-10-04T08:13:37,751 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=7cf4420553a66064d10b73e18ff7ce65, ASSIGN 2023-10-04T08:13:37,752 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=7cf4420553a66064d10b73e18ff7ce65, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:37,903 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=7cf4420553a66064d10b73e18ff7ce65, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:37,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 7cf4420553a66064d10b73e18ff7ce65, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:38,064 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:38,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => 7cf4420553a66064d10b73e18ff7ce65, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:38,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:38,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,066 INFO [StoreOpener-7cf4420553a66064d10b73e18ff7ce65-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,067 INFO [StoreOpener-7cf4420553a66064d10b73e18ff7ce65-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cf4420553a66064d10b73e18ff7ce65 columnFamilyName testFamily 2023-10-04T08:13:38,068 DEBUG [StoreOpener-7cf4420553a66064d10b73e18ff7ce65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:38,068 INFO [StoreOpener-7cf4420553a66064d10b73e18ff7ce65-1 {}] regionserver.HStore(324): Store=7cf4420553a66064d10b73e18ff7ce65/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:38,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,070 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:13:38,074 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:38,075 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1093): Opened 7cf4420553a66064d10b73e18ff7ce65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72146045, jitterRate=0.07505984604358673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:38,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(992): Region open journal for 7cf4420553a66064d10b73e18ff7ce65: 2023-10-04T08:13:38,076 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65., pid=59, masterSystemTime=1696407218057 2023-10-04T08:13:38,078 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:38,078 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:13:38,078 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=7cf4420553a66064d10b73e18ff7ce65, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:38,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-10-04T08:13:38,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 7cf4420553a66064d10b73e18ff7ce65, server=jenkins-hbase3.apache.org,45997,1696407041329 in 175 msec 2023-10-04T08:13:38,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-10-04T08:13:38,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=7cf4420553a66064d10b73e18ff7ce65, ASSIGN in 332 msec 2023-10-04T08:13:38,084 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:38,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407218084"}]},"ts":"1696407218084"} 2023-10-04T08:13:38,086 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:38,088 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:38,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 781 msec 2023-10-04T08:13:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-04T08:13:38,313 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2023-10-04T08:13:38,313 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:38,313 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:38,317 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:38,317 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:38,317 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2023-10-04T08:13:38,320 DEBUG [hconnection-0x8d967e3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:38,322 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:38,330 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:38,332 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:13:38,332 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:13:38,333 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x49154ac6 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:13:38,333 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x49154ac6 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:38,335 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37854, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:13:38,338 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:13:38,338 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885230 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:38,339 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:13:38,339 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885230 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:38,339 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-10-04T08:13:41,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2023-10-04T08:13:41,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:41,783 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:13:43,596 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:43,712 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2023-10-04T08:13:45,281 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666431 expired clientIPAndPort=172.31.12.81:46462, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:13:47,340 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9001 2023-10-04T08:13:47,340 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885230 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:13:47,340 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7fbf50c4 to 127.0.0.1:50391 2023-10-04T08:13:47,341 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:47,341 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666386' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-04T08:13:47,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:37854 deadline: 1696407233340, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666386' does not exist 2023-10-04T08:13:47,341 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:47,341 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:47,341 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x49154ac6 to 127.0.0.1:50391 2023-10-04T08:13:47,341 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,45997,1696407041329 for closing scanner id = -452929323842666386 for 7cf4420553a66064d10b73e18ff7ce65 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666386' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-04T08:13:47,353 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=214 (was 212) - Thread LEAK? -, OpenFileDescriptor=482 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=142 (was 140) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=6177 (was 6181) 2023-10-04T08:13:47,363 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=142, ProcessCount=170, AvailableMemoryMB=6177 2023-10-04T08:13:47,365 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x505411ef to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:47,369 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x505411ef {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc67776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:47,370 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1e47f2da to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:47,374 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b6a64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:47,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2023-10-04T08:13:47,379 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:47,379 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:47,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2023-10-04T08:13:47,380 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-04T08:13:47,794 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 87c60d1bc9734410abc9413183bad22a, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:47,840 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 87c60d1bc9734410abc9413183bad22a, disabling compactions & flushes 2023-10-04T08:13:48,205 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. after waiting 0 ms 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,205 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,205 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 87c60d1bc9734410abc9413183bad22a: 2023-10-04T08:13:48,207 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:48,207 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1696407228207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407228207"}]},"ts":"1696407228207"} 2023-10-04T08:13:48,209 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:48,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:48,210 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407228209"}]},"ts":"1696407228209"} 2023-10-04T08:13:48,211 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:48,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=87c60d1bc9734410abc9413183bad22a, ASSIGN}] 2023-10-04T08:13:48,222 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=87c60d1bc9734410abc9413183bad22a, ASSIGN 2023-10-04T08:13:48,223 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=87c60d1bc9734410abc9413183bad22a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:48,375 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=87c60d1bc9734410abc9413183bad22a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:48,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 87c60d1bc9734410abc9413183bad22a, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-04T08:13:48,535 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7254): Opening region: {ENCODED => 87c60d1bc9734410abc9413183bad22a, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:48,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:48,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7296): checking encryption for 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7299): checking classloading for 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,538 INFO [StoreOpener-87c60d1bc9734410abc9413183bad22a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,539 INFO [StoreOpener-87c60d1bc9734410abc9413183bad22a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87c60d1bc9734410abc9413183bad22a columnFamilyName testFamily 2023-10-04T08:13:48,539 DEBUG [StoreOpener-87c60d1bc9734410abc9413183bad22a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:48,540 INFO [StoreOpener-87c60d1bc9734410abc9413183bad22a-1 {}] regionserver.HStore(324): Store=87c60d1bc9734410abc9413183bad22a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:48,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1076): writing seq id for 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:13:48,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:48,547 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1093): Opened 87c60d1bc9734410abc9413183bad22a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67945705, jitterRate=0.012469902634620667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:48,547 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(992): Region open journal for 87c60d1bc9734410abc9413183bad22a: 2023-10-04T08:13:48,548 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2556): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a., pid=62, masterSystemTime=1696407228530 2023-10-04T08:13:48,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2583): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,550 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:13:48,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=87c60d1bc9734410abc9413183bad22a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:48,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=62, resume processing ppid=61 2023-10-04T08:13:48,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 87c60d1bc9734410abc9413183bad22a, server=jenkins-hbase3.apache.org,45997,1696407041329 in 175 msec 2023-10-04T08:13:48,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-10-04T08:13:48,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=87c60d1bc9734410abc9413183bad22a, ASSIGN in 334 msec 2023-10-04T08:13:48,556 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:48,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407228556"}]},"ts":"1696407228556"} 2023-10-04T08:13:48,557 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:48,559 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:48,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1820 sec 2023-10-04T08:13:49,713 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2023-10-04T08:13:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-04T08:13:50,385 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2023-10-04T08:13:50,385 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:50,385 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:50,388 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:50,388 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:50,388 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2023-10-04T08:13:50,390 DEBUG [hconnection-0x4ac3375f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:50,392 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:50,400 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:50,402 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2023-10-04T08:13:50,403 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2023-10-04T08:13:50,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885234 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,405 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2023-10-04T08:13:50,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885234 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666382 2023-10-04T08:13:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:53346 deadline: 1696407233405, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:50,407 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:13:50,411 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2023-10-04T08:13:50,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885235 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,412 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2023-10-04T08:13:50,412 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:50,413 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:50,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885236 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-452929323842666380 2023-10-04T08:13:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:53346 deadline: 1696407233414, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:50,415 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:13:50,417 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:50,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885237 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-452929323842666379 2023-10-04T08:13:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:53346 deadline: 1696407233417, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:50,418 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:13:50,420 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:50,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885238 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:13:50,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-452929323842666378 2023-10-04T08:13:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:53346 deadline: 1696407233420, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2023-10-04T08:13:50,421 DEBUG [Listener at localhost/43533 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:13:50,423 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2023-10-04T08:13:50,424 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1e47f2da to 127.0.0.1:50391 2023-10-04T08:13:50,424 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:50,424 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:50,424 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:50,424 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:50,424 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x505411ef to 127.0.0.1:50391 2023-10-04T08:13:50,435 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=216 (was 214) - Thread LEAK? -, OpenFileDescriptor=484 (was 482) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=138 (was 142), ProcessCount=170 (was 170), AvailableMemoryMB=6171 (was 6177) 2023-10-04T08:13:50,446 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=216, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=138, ProcessCount=170, AvailableMemoryMB=6171 2023-10-04T08:13:50,447 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x629245a2 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:50,451 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x629245a2 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f08115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:50,452 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x345a1e6f to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:50,455 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:50,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2023-10-04T08:13:50,460 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:50,460 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:50,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2023-10-04T08:13:50,461 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-10-04T08:13:50,874 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 6438062196d04176b2d1947e41c0d3e6, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:51,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2023-10-04T08:13:51,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:13:51,285 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:51,285 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 6438062196d04176b2d1947e41c0d3e6, disabling compactions & flushes 2023-10-04T08:13:51,285 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,285 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,285 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. after waiting 0 ms 2023-10-04T08:13:51,285 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,286 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 6438062196d04176b2d1947e41c0d3e6: 2023-10-04T08:13:51,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:51,287 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1696407231287"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407231287"}]},"ts":"1696407231287"} 2023-10-04T08:13:51,289 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:51,289 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:51,290 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407231290"}]},"ts":"1696407231290"} 2023-10-04T08:13:51,291 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:51,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=6438062196d04176b2d1947e41c0d3e6, ASSIGN}] 2023-10-04T08:13:51,299 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=6438062196d04176b2d1947e41c0d3e6, ASSIGN 2023-10-04T08:13:51,300 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=6438062196d04176b2d1947e41c0d3e6, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:51,451 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=6438062196d04176b2d1947e41c0d3e6, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:51,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 6438062196d04176b2d1947e41c0d3e6, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-10-04T08:13:51,609 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7254): Opening region: {ENCODED => 6438062196d04176b2d1947e41c0d3e6, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:51,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:51,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7296): checking encryption for 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7299): checking classloading for 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,611 INFO [StoreOpener-6438062196d04176b2d1947e41c0d3e6-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,613 INFO [StoreOpener-6438062196d04176b2d1947e41c0d3e6-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6438062196d04176b2d1947e41c0d3e6 columnFamilyName testFamily 2023-10-04T08:13:51,613 DEBUG [StoreOpener-6438062196d04176b2d1947e41c0d3e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:51,614 INFO [StoreOpener-6438062196d04176b2d1947e41c0d3e6-1 {}] regionserver.HStore(324): Store=6438062196d04176b2d1947e41c0d3e6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:51,614 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,617 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1076): writing seq id for 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:13:51,619 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:51,620 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): Opened 6438062196d04176b2d1947e41c0d3e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68312385, jitterRate=0.017933860421180725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:51,620 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(992): Region open journal for 6438062196d04176b2d1947e41c0d3e6: 2023-10-04T08:13:51,621 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6., pid=65, masterSystemTime=1696407231605 2023-10-04T08:13:51,623 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,623 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:13:51,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=6438062196d04176b2d1947e41c0d3e6, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:51,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-10-04T08:13:51,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 6438062196d04176b2d1947e41c0d3e6, server=jenkins-hbase3.apache.org,45997,1696407041329 in 172 msec 2023-10-04T08:13:51,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=64, resume processing ppid=63 2023-10-04T08:13:51,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=6438062196d04176b2d1947e41c0d3e6, ASSIGN in 329 msec 2023-10-04T08:13:51,629 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:51,629 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407231629"}]},"ts":"1696407231629"} 2023-10-04T08:13:51,630 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:51,633 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:51,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 1.1760 sec 2023-10-04T08:13:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=63 2023-10-04T08:13:53,465 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2023-10-04T08:13:53,465 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:53,465 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:53,468 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:53,468 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:53,469 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2023-10-04T08:13:53,471 DEBUG [hconnection-0x33de240c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:53,473 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53360, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:53,479 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:53,481 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:13:53,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:13:53,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2023-10-04T08:13:54,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:13:54,890 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666427 expired clientIPAndPort=172.31.12.81:46466, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 12 bytes 2023-10-04T08:13:55,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:13:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:53360 deadline: 1696407236483, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:13:55,505 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:13:55,506 DEBUG [Listener at localhost/43533 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885243 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:13:55,507 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2026 2023-10-04T08:13:55,508 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x345a1e6f to 127.0.0.1:50391 2023-10-04T08:13:55,508 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:55,508 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:13:55,508 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:13:55,508 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:13:55,508 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x629245a2 to 127.0.0.1:50391 2023-10-04T08:13:55,520 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=216 (was 216), OpenFileDescriptor=487 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=135 (was 138), ProcessCount=170 (was 170), AvailableMemoryMB=6158 (was 6171) 2023-10-04T08:13:55,530 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=216, OpenFileDescriptor=487, MaxFileDescriptor=60000, SystemLoadAverage=135, ProcessCount=170, AvailableMemoryMB=6158 2023-10-04T08:13:55,532 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7f6cc42f to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:55,536 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x7f6cc42f {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b64b8bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:55,536 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1bc34d79 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:13:55,544 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e42ff4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:13:55,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:13:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2023-10-04T08:13:55,548 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:13:55,548 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:55,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2023-10-04T08:13:55,549 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:13:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-04T08:13:55,714 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2023-10-04T08:13:55,959 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1bcae92c528d1fed7f31bf8de10093d1, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 1bcae92c528d1fed7f31bf8de10093d1, disabling compactions & flushes 2023-10-04T08:13:56,372 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. after waiting 0 ms 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,372 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,372 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1bcae92c528d1fed7f31bf8de10093d1: 2023-10-04T08:13:56,374 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:13:56,374 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1696407236374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407236374"}]},"ts":"1696407236374"} 2023-10-04T08:13:56,376 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:13:56,376 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:13:56,376 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407236376"}]},"ts":"1696407236376"} 2023-10-04T08:13:56,378 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:13:56,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=1bcae92c528d1fed7f31bf8de10093d1, ASSIGN}] 2023-10-04T08:13:56,383 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=1bcae92c528d1fed7f31bf8de10093d1, ASSIGN 2023-10-04T08:13:56,384 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=1bcae92c528d1fed7f31bf8de10093d1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:13:56,535 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=1bcae92c528d1fed7f31bf8de10093d1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:56,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 1bcae92c528d1fed7f31bf8de10093d1, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:13:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-04T08:13:56,693 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7254): Opening region: {ENCODED => 1bcae92c528d1fed7f31bf8de10093d1, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:13:56,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:13:56,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7296): checking encryption for 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7299): checking classloading for 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,696 INFO [StoreOpener-1bcae92c528d1fed7f31bf8de10093d1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,697 INFO [StoreOpener-1bcae92c528d1fed7f31bf8de10093d1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bcae92c528d1fed7f31bf8de10093d1 columnFamilyName testFamily 2023-10-04T08:13:56,697 DEBUG [StoreOpener-1bcae92c528d1fed7f31bf8de10093d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:13:56,698 INFO [StoreOpener-1bcae92c528d1fed7f31bf8de10093d1-1 {}] regionserver.HStore(324): Store=1bcae92c528d1fed7f31bf8de10093d1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:13:56,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1076): writing seq id for 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:13:56,703 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:13:56,704 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1093): Opened 1bcae92c528d1fed7f31bf8de10093d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59911013, jitterRate=-0.10725633800029755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:13:56,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(992): Region open journal for 1bcae92c528d1fed7f31bf8de10093d1: 2023-10-04T08:13:56,705 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1., pid=68, masterSystemTime=1696407236689 2023-10-04T08:13:56,707 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,707 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:13:56,707 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=1bcae92c528d1fed7f31bf8de10093d1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:13:56,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-10-04T08:13:56,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 1bcae92c528d1fed7f31bf8de10093d1, server=jenkins-hbase3.apache.org,45997,1696407041329 in 172 msec 2023-10-04T08:13:56,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-10-04T08:13:56,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=1bcae92c528d1fed7f31bf8de10093d1, ASSIGN in 328 msec 2023-10-04T08:13:56,713 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:13:56,713 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407236713"}]},"ts":"1696407236713"} 2023-10-04T08:13:56,714 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:13:56,716 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:13:56,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1700 sec 2023-10-04T08:13:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-04T08:13:58,553 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2023-10-04T08:13:58,553 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:13:58,553 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:58,557 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2023-10-04T08:13:58,557 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:13:58,557 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2023-10-04T08:13:58,559 DEBUG [hconnection-0x59038f16-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:13:58,560 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:13:58,567 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:13:58,569 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:13:58,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:14:00,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 6, skipping over this response of 89 bytes 2023-10-04T08:14:00,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:14:01,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2023-10-04T08:14:01,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:01,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2023-10-04T08:14:01,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:01,580 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5014: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5014: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:14:01,580 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2023-10-04T08:14:01.580Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5014: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5014: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=981ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:14:01,581 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3011 2023-10-04T08:14:01,581 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1bc34d79 to 127.0.0.1:50391 2023-10-04T08:14:01,581 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:01,581 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:01,581 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:01,581 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:01,582 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7f6cc42f to 127.0.0.1:50391 2023-10-04T08:14:01,593 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=214 (was 216), OpenFileDescriptor=486 (was 487), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=132 (was 135), ProcessCount=171 (was 170) - ProcessCount LEAK? -, AvailableMemoryMB=6147 (was 6158) 2023-10-04T08:14:01,603 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=214, OpenFileDescriptor=486, MaxFileDescriptor=60000, SystemLoadAverage=132, ProcessCount=171, AvailableMemoryMB=6146 2023-10-04T08:14:01,604 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c23fab5 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:01,608 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x4c23fab5 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163c7492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:01,609 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7a496d7e to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:01,613 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@353b70db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:01,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2023-10-04T08:14:01,617 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:01,617 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:01,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2023-10-04T08:14:01,618 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-10-04T08:14:02,031 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 59fbb4b736d248cd85fe604c5ee58655, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:02,090 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:14:02,196 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:02,231 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 59fbb4b736d248cd85fe604c5ee58655, disabling compactions & flushes 2023-10-04T08:14:02,440 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. after waiting 0 ms 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,440 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,440 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 59fbb4b736d248cd85fe604c5ee58655: 2023-10-04T08:14:02,441 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:02,441 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1696407242441"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407242441"}]},"ts":"1696407242441"} 2023-10-04T08:14:02,443 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:02,444 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:02,444 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407242444"}]},"ts":"1696407242444"} 2023-10-04T08:14:02,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:02,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=59fbb4b736d248cd85fe604c5ee58655, ASSIGN}] 2023-10-04T08:14:02,451 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=59fbb4b736d248cd85fe604c5ee58655, ASSIGN 2023-10-04T08:14:02,451 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=59fbb4b736d248cd85fe604c5ee58655, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:02,602 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=59fbb4b736d248cd85fe604c5ee58655, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:02,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 59fbb4b736d248cd85fe604c5ee58655, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-10-04T08:14:02,761 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7254): Opening region: {ENCODED => 59fbb4b736d248cd85fe604c5ee58655, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:02,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:02,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7296): checking encryption for 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7299): checking classloading for 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,763 INFO [StoreOpener-59fbb4b736d248cd85fe604c5ee58655-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,764 INFO [StoreOpener-59fbb4b736d248cd85fe604c5ee58655-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fbb4b736d248cd85fe604c5ee58655 columnFamilyName testFamily 2023-10-04T08:14:02,764 DEBUG [StoreOpener-59fbb4b736d248cd85fe604c5ee58655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:02,765 INFO [StoreOpener-59fbb4b736d248cd85fe604c5ee58655-1 {}] regionserver.HStore(324): Store=59fbb4b736d248cd85fe604c5ee58655/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:02,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,768 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1076): writing seq id for 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:14:02,770 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:02,771 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1093): Opened 59fbb4b736d248cd85fe604c5ee58655; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63603376, jitterRate=-0.05223584175109863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:02,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(992): Region open journal for 59fbb4b736d248cd85fe604c5ee58655: 2023-10-04T08:14:02,772 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655., pid=71, masterSystemTime=1696407242756 2023-10-04T08:14:02,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,774 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:14:02,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=59fbb4b736d248cd85fe604c5ee58655, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:02,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-10-04T08:14:02,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 59fbb4b736d248cd85fe604c5ee58655, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:02,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-10-04T08:14:02,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=59fbb4b736d248cd85fe604c5ee58655, ASSIGN in 329 msec 2023-10-04T08:14:02,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:02,780 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407242780"}]},"ts":"1696407242780"} 2023-10-04T08:14:02,781 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:02,784 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:02,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1690 sec 2023-10-04T08:14:04,498 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666421 expired clientIPAndPort=172.31.12.81:36392, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:14:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=69 2023-10-04T08:14:04,622 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2023-10-04T08:14:04,622 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:04,622 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:04,625 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:04,625 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:04,625 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2023-10-04T08:14:04,627 DEBUG [hconnection-0x51ff9aff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:04,629 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:04,636 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:04,637 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:04,638 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:14:04,639 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x4c23fab5 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:14:04,639 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x4c23fab5 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:04,641 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:14:04,644 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:14:04,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885255 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:04,645 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:14:04,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885255 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:04,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-10-04T08:14:07,650 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3004 2023-10-04T08:14:07,650 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7a496d7e to 127.0.0.1:50391 2023-10-04T08:14:07,650 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885255 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:14:07,650 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:07,650 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:07,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666361' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3586) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-04T08:14:07,650 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:07,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:37544 deadline: 1696407248649, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-452929323842666361' does not exist 2023-10-04T08:14:07,650 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:07,650 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c23fab5 to 127.0.0.1:50391 2023-10-04T08:14:07,651 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,45997,1696407041329 for closing scanner id = -452929323842666361 for 59fbb4b736d248cd85fe604c5ee58655 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:229) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:237) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2023-10-04T08:14:07,663 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=214 (was 214), OpenFileDescriptor=483 (was 486), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=138 (was 132) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=6134 (was 6146) 2023-10-04T08:14:07,673 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=214, OpenFileDescriptor=483, MaxFileDescriptor=60000, SystemLoadAverage=138, ProcessCount=171, AvailableMemoryMB=6134 2023-10-04T08:14:07,675 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5c7c43c9 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:07,681 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x5c7c43c9 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76639b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:07,682 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x77db518c to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:07,688 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcd846a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:07,690 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:07,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2023-10-04T08:14:07,692 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:07,693 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:07,693 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2023-10-04T08:14:07,693 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-04T08:14:08,107 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b535ccf633612976045f65cd48c88d1b, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:08,146 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:14:08,263 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:08,301 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing b535ccf633612976045f65cd48c88d1b, disabling compactions & flushes 2023-10-04T08:14:08,517 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. after waiting 0 ms 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,517 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,517 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for b535ccf633612976045f65cd48c88d1b: 2023-10-04T08:14:08,518 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:08,519 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1696407248518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407248518"}]},"ts":"1696407248518"} 2023-10-04T08:14:08,520 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:08,521 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:08,521 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407248521"}]},"ts":"1696407248521"} 2023-10-04T08:14:08,523 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:08,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=b535ccf633612976045f65cd48c88d1b, ASSIGN}] 2023-10-04T08:14:08,529 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=b535ccf633612976045f65cd48c88d1b, ASSIGN 2023-10-04T08:14:08,530 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=b535ccf633612976045f65cd48c88d1b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:08,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=b535ccf633612976045f65cd48c88d1b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:08,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure b535ccf633612976045f65cd48c88d1b, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-04T08:14:08,840 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => b535ccf633612976045f65cd48c88d1b, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:08,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:08,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,842 INFO [StoreOpener-b535ccf633612976045f65cd48c88d1b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,843 INFO [StoreOpener-b535ccf633612976045f65cd48c88d1b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b535ccf633612976045f65cd48c88d1b columnFamilyName testFamily 2023-10-04T08:14:08,844 DEBUG [StoreOpener-b535ccf633612976045f65cd48c88d1b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:08,844 INFO [StoreOpener-b535ccf633612976045f65cd48c88d1b-1 {}] regionserver.HStore(324): Store=b535ccf633612976045f65cd48c88d1b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:08,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for b535ccf633612976045f65cd48c88d1b 2023-10-04T08:14:08,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:08,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1093): Opened b535ccf633612976045f65cd48c88d1b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73494949, jitterRate=0.09516008198261261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:08,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(992): Region open journal for b535ccf633612976045f65cd48c88d1b: 2023-10-04T08:14:08,851 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b., pid=74, masterSystemTime=1696407248835 2023-10-04T08:14:08,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,853 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:14:08,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=b535ccf633612976045f65cd48c88d1b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:08,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-10-04T08:14:08,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure b535ccf633612976045f65cd48c88d1b, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:08,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-10-04T08:14:08,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=b535ccf633612976045f65cd48c88d1b, ASSIGN in 328 msec 2023-10-04T08:14:08,858 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:08,859 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407248858"}]},"ts":"1696407248858"} 2023-10-04T08:14:08,860 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:08,862 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:08,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1710 sec 2023-10-04T08:14:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-04T08:14:10,697 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2023-10-04T08:14:10,697 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:10,698 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:10,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32809 bytes) of info 2023-10-04T08:14:10,704 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885256 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:10,706 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:10,706 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:10,706 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2023-10-04T08:14:10,708 DEBUG [hconnection-0x189345c6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:10,709 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36882, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:10,716 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:10,718 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:10,719 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885259 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:14:10,719 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2023-10-04T08:14:11,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2023-10-04T08:14:11,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2023-10-04T08:14:11,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:11,504 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666419 expired clientIPAndPort=172.31.12.81:36392, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:14:14,342 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:14,383 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2023-10-04T08:14:16,608 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666414 expired clientIPAndPort=172.31.12.81:54376, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:14:17,220 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2023-10-04T08:14:17,220 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x77db518c to 127.0.0.1:50391 2023-10-04T08:14:17,221 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:17,221 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:17,221 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:17,221 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:17,221 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5c7c43c9 to 127.0.0.1:50391 2023-10-04T08:14:17,234 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 483), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=132 (was 138), ProcessCount=171 (was 171), AvailableMemoryMB=6104 (was 6134) 2023-10-04T08:14:17,244 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=132, ProcessCount=171, AvailableMemoryMB=6103 2023-10-04T08:14:17,245 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2323659d to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:17,249 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x2323659d {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ea273f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:17,249 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x53503404 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:17,253 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9e82b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:17,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2023-10-04T08:14:17,257 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:17,257 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:17,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2023-10-04T08:14:17,258 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-04T08:14:17,670 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9fe8cf8cd62090355177a9f2de04a723, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 9fe8cf8cd62090355177a9f2de04a723, disabling compactions & flushes 2023-10-04T08:14:18,080 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. after waiting 0 ms 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,080 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,080 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9fe8cf8cd62090355177a9f2de04a723: 2023-10-04T08:14:18,081 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:18,081 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1696407258081"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407258081"}]},"ts":"1696407258081"} 2023-10-04T08:14:18,083 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:18,084 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:18,084 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407258084"}]},"ts":"1696407258084"} 2023-10-04T08:14:18,085 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:18,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9fe8cf8cd62090355177a9f2de04a723, ASSIGN}] 2023-10-04T08:14:18,091 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9fe8cf8cd62090355177a9f2de04a723, ASSIGN 2023-10-04T08:14:18,092 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9fe8cf8cd62090355177a9f2de04a723, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:18,243 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=9fe8cf8cd62090355177a9f2de04a723, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:18,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 9fe8cf8cd62090355177a9f2de04a723, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-04T08:14:18,401 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,401 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7254): Opening region: {ENCODED => 9fe8cf8cd62090355177a9f2de04a723, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:18,402 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,402 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:18,402 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7296): checking encryption for 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,402 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7299): checking classloading for 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,403 INFO [StoreOpener-9fe8cf8cd62090355177a9f2de04a723-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,405 INFO [StoreOpener-9fe8cf8cd62090355177a9f2de04a723-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fe8cf8cd62090355177a9f2de04a723 columnFamilyName testFamily 2023-10-04T08:14:18,405 DEBUG [StoreOpener-9fe8cf8cd62090355177a9f2de04a723-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:18,405 INFO [StoreOpener-9fe8cf8cd62090355177a9f2de04a723-1 {}] regionserver.HStore(324): Store=9fe8cf8cd62090355177a9f2de04a723/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:18,406 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,406 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,408 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1076): writing seq id for 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:14:18,410 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:18,411 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): Opened 9fe8cf8cd62090355177a9f2de04a723; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74314973, jitterRate=0.10737939178943634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:18,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(992): Region open journal for 9fe8cf8cd62090355177a9f2de04a723: 2023-10-04T08:14:18,412 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723., pid=77, masterSystemTime=1696407258397 2023-10-04T08:14:18,414 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,414 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:14:18,414 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=9fe8cf8cd62090355177a9f2de04a723, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:18,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-10-04T08:14:18,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 9fe8cf8cd62090355177a9f2de04a723, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:18,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-10-04T08:14:18,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9fe8cf8cd62090355177a9f2de04a723, ASSIGN in 328 msec 2023-10-04T08:14:18,420 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:18,420 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407258420"}]},"ts":"1696407258420"} 2023-10-04T08:14:18,421 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:18,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:18,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1690 sec 2023-10-04T08:14:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-04T08:14:20,262 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2023-10-04T08:14:20,262 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:20,262 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:20,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32886 bytes) of info 2023-10-04T08:14:20,266 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885260 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:20,267 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:20,267 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:20,267 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2023-10-04T08:14:20,269 DEBUG [hconnection-0x220c0ce1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:20,271 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49200, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:20,278 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:20,279 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:20,281 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885263 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2023-10-04T08:14:20,281 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2023-10-04T08:14:20,384 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2023-10-04T08:14:20,612 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666409 expired clientIPAndPort=172.31.12.81:46962, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:14:21,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2023-10-04T08:14:21,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:21,782 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2023-10-04T08:14:21,783 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x53503404 to 127.0.0.1:50391 2023-10-04T08:14:21,783 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:21,783 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:21,783 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:21,783 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:21,783 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2323659d to 127.0.0.1:50391 2023-10-04T08:14:21,794 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=217 (was 214) - Thread LEAK? -, OpenFileDescriptor=484 (was 482) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=129 (was 132), ProcessCount=171 (was 171), AvailableMemoryMB=6097 (was 6103) 2023-10-04T08:14:21,804 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=217, OpenFileDescriptor=484, MaxFileDescriptor=60000, SystemLoadAverage=129, ProcessCount=171, AvailableMemoryMB=6096 2023-10-04T08:14:21,806 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0c558a24 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:21,809 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x0c558a24 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dfcfba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:21,810 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20117135 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:21,814 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@222c5154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:21,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2023-10-04T08:14:21,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:21,818 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:21,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2023-10-04T08:14:21,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-04T08:14:22,229 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 17f6bc2dc04de9b433ce6efaae857bb1, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 17f6bc2dc04de9b433ce6efaae857bb1, disabling compactions & flushes 2023-10-04T08:14:22,640 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. after waiting 0 ms 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,640 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,640 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 17f6bc2dc04de9b433ce6efaae857bb1: 2023-10-04T08:14:22,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:22,641 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1696407262641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407262641"}]},"ts":"1696407262641"} 2023-10-04T08:14:22,643 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:22,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:22,644 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407262644"}]},"ts":"1696407262644"} 2023-10-04T08:14:22,645 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:22,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=17f6bc2dc04de9b433ce6efaae857bb1, ASSIGN}] 2023-10-04T08:14:22,650 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=17f6bc2dc04de9b433ce6efaae857bb1, ASSIGN 2023-10-04T08:14:22,651 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=17f6bc2dc04de9b433ce6efaae857bb1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:22,713 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666407 expired clientIPAndPort=172.31.12.81:46962, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:14:22,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=17f6bc2dc04de9b433ce6efaae857bb1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:22,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 17f6bc2dc04de9b433ce6efaae857bb1, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-04T08:14:22,960 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,960 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7254): Opening region: {ENCODED => 17f6bc2dc04de9b433ce6efaae857bb1, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:22,960 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,960 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:22,961 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7296): checking encryption for 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,961 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7299): checking classloading for 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,962 INFO [StoreOpener-17f6bc2dc04de9b433ce6efaae857bb1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,964 INFO [StoreOpener-17f6bc2dc04de9b433ce6efaae857bb1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17f6bc2dc04de9b433ce6efaae857bb1 columnFamilyName testFamily 2023-10-04T08:14:22,964 DEBUG [StoreOpener-17f6bc2dc04de9b433ce6efaae857bb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:22,964 INFO [StoreOpener-17f6bc2dc04de9b433ce6efaae857bb1-1 {}] regionserver.HStore(324): Store=17f6bc2dc04de9b433ce6efaae857bb1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:22,965 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,965 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,967 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1076): writing seq id for 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:14:22,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:22,970 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): Opened 17f6bc2dc04de9b433ce6efaae857bb1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64037763, jitterRate=-0.04576297104358673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:22,971 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(992): Region open journal for 17f6bc2dc04de9b433ce6efaae857bb1: 2023-10-04T08:14:22,971 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1., pid=80, masterSystemTime=1696407262955 2023-10-04T08:14:22,973 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,973 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:14:22,973 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=17f6bc2dc04de9b433ce6efaae857bb1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:22,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-10-04T08:14:22,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 17f6bc2dc04de9b433ce6efaae857bb1, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=79, resume processing ppid=78 2023-10-04T08:14:22,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=17f6bc2dc04de9b433ce6efaae857bb1, ASSIGN in 327 msec 2023-10-04T08:14:22,978 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:22,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407262978"}]},"ts":"1696407262978"} 2023-10-04T08:14:22,980 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:22,982 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:22,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1660 sec 2023-10-04T08:14:23,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-04T08:14:24,822 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2023-10-04T08:14:24,822 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:24,822 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:24,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32859 bytes) of info 2023-10-04T08:14:24,826 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885264 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:24,827 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:24,828 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:24,828 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2023-10-04T08:14:24,830 DEBUG [hconnection-0x6a753ca4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:24,831 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49210, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:24,838 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:24,840 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:24,842 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:14:24,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885267 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:24,843 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:14:24,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885267 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:24,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2023-10-04T08:14:26,385 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2023-10-04T08:14:27,851 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4017: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3006ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:14:27,852 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3008 2023-10-04T08:14:27,852 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x20117135 to 127.0.0.1:50391 2023-10-04T08:14:27,852 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:27,852 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:27,852 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:27,852 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:27,852 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0c558a24 to 127.0.0.1:50391 2023-10-04T08:14:27,865 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=214 (was 217), OpenFileDescriptor=485 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=125 (was 129), ProcessCount=171 (was 171), AvailableMemoryMB=6046 (was 6096) 2023-10-04T08:14:27,875 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=214, OpenFileDescriptor=485, MaxFileDescriptor=60000, SystemLoadAverage=125, ProcessCount=171, AvailableMemoryMB=6046 2023-10-04T08:14:27,876 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x68ab1e74 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:27,884 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x68ab1e74 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63cd063c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:27,884 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e359cfa to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:27,888 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad1ec5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:27,890 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:27,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2023-10-04T08:14:27,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:27,892 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:27,892 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2023-10-04T08:14:27,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-04T08:14:28,304 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4958717ea3998bbee14b70ccbd8d2b06, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:28,344 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:14:28,713 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:28,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 4958717ea3998bbee14b70ccbd8d2b06, disabling compactions & flushes 2023-10-04T08:14:28,714 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:28,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:28,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. after waiting 0 ms 2023-10-04T08:14:28,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:28,714 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:28,714 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4958717ea3998bbee14b70ccbd8d2b06: 2023-10-04T08:14:28,715 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:28,715 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1696407268715"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407268715"}]},"ts":"1696407268715"} 2023-10-04T08:14:28,717 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:28,718 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:28,718 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407268718"}]},"ts":"1696407268718"} 2023-10-04T08:14:28,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:28,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=4958717ea3998bbee14b70ccbd8d2b06, ASSIGN}] 2023-10-04T08:14:28,724 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=4958717ea3998bbee14b70ccbd8d2b06, ASSIGN 2023-10-04T08:14:28,725 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=4958717ea3998bbee14b70ccbd8d2b06, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:28,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=4958717ea3998bbee14b70ccbd8d2b06, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:28,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 4958717ea3998bbee14b70ccbd8d2b06, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-04T08:14:29,034 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:29,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 4958717ea3998bbee14b70ccbd8d2b06, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:29,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:29,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,034 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,036 INFO [StoreOpener-4958717ea3998bbee14b70ccbd8d2b06-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,037 INFO [StoreOpener-4958717ea3998bbee14b70ccbd8d2b06-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4958717ea3998bbee14b70ccbd8d2b06 columnFamilyName testFamily 2023-10-04T08:14:29,037 DEBUG [StoreOpener-4958717ea3998bbee14b70ccbd8d2b06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:29,038 INFO [StoreOpener-4958717ea3998bbee14b70ccbd8d2b06-1 {}] regionserver.HStore(324): Store=4958717ea3998bbee14b70ccbd8d2b06/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:29,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:14:29,043 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:29,044 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1093): Opened 4958717ea3998bbee14b70ccbd8d2b06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72510676, jitterRate=0.0804932713508606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:29,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 4958717ea3998bbee14b70ccbd8d2b06: 2023-10-04T08:14:29,045 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06., pid=83, masterSystemTime=1696407269029 2023-10-04T08:14:29,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:29,047 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:14:29,047 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=4958717ea3998bbee14b70ccbd8d2b06, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:29,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-10-04T08:14:29,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 4958717ea3998bbee14b70ccbd8d2b06, server=jenkins-hbase3.apache.org,45997,1696407041329 in 170 msec 2023-10-04T08:14:29,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-10-04T08:14:29,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=4958717ea3998bbee14b70ccbd8d2b06, ASSIGN in 327 msec 2023-10-04T08:14:29,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:29,052 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407269052"}]},"ts":"1696407269052"} 2023-10-04T08:14:29,053 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:29,056 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:29,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1660 sec 2023-10-04T08:14:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-04T08:14:30,896 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2023-10-04T08:14:30,896 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:30,896 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:30,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2023-10-04T08:14:30,900 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885268 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:30,902 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:30,902 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:30,902 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2023-10-04T08:14:30,904 DEBUG [hconnection-0x2da7ad3a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:30,906 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:30,914 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:30,916 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:30,918 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:14:30,919 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2023-10-04T08:14:31,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2023-10-04T08:14:31,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:31,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2023-10-04T08:14:31,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:32,386 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2023-10-04T08:14:34,594 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:34,823 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666393 expired clientIPAndPort=172.31.12.81:44470, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:14:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 7, skipping over this response of 12 bytes 2023-10-04T08:14:37,934 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:14:37,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.31.12.81:35918 deadline: 1696407279918, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2023-10-04T08:14:37,935 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2023-10-04T08:14:37,936 DEBUG [Listener at localhost/43533 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 17993814749866885271 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3211) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3594) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:44996) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-04T08:14:37,937 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7021 2023-10-04T08:14:37,938 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e359cfa to 127.0.0.1:50391 2023-10-04T08:14:37,938 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:37,938 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:37,938 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:37,938 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:37,938 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x68ab1e74 to 127.0.0.1:50391 2023-10-04T08:14:37,951 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 485), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=121 (was 125), ProcessCount=167 (was 171), AvailableMemoryMB=6029 (was 6046) 2023-10-04T08:14:37,961 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=214, OpenFileDescriptor=481, MaxFileDescriptor=60000, SystemLoadAverage=121, ProcessCount=167, AvailableMemoryMB=6029 2023-10-04T08:14:37,962 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5f8b1bc7 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:37,966 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x5f8b1bc7 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3853f017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:37,966 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x224eba2a to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:37,970 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9c7c03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:37,972 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:37,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2023-10-04T08:14:37,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:37,974 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:37,974 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2023-10-04T08:14:37,975 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=84 2023-10-04T08:14:37,990 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a86ff5fbeda7bbb8ff18cf9e11cfdf8e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing a86ff5fbeda7bbb8ff18cf9e11cfdf8e, disabling compactions & flushes 2023-10-04T08:14:38,401 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. after waiting 0 ms 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,401 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,401 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for a86ff5fbeda7bbb8ff18cf9e11cfdf8e: 2023-10-04T08:14:38,402 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:38,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1696407278402"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407278402"}]},"ts":"1696407278402"} 2023-10-04T08:14:38,404 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:38,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:38,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407278405"}]},"ts":"1696407278405"} 2023-10-04T08:14:38,406 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:38,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, ASSIGN}] 2023-10-04T08:14:38,413 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, ASSIGN 2023-10-04T08:14:38,413 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:38,564 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:38,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure a86ff5fbeda7bbb8ff18cf9e11cfdf8e, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:38,722 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7254): Opening region: {ENCODED => a86ff5fbeda7bbb8ff18cf9e11cfdf8e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:38,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:38,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7296): checking encryption for a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7299): checking classloading for a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,724 INFO [StoreOpener-a86ff5fbeda7bbb8ff18cf9e11cfdf8e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,726 INFO [StoreOpener-a86ff5fbeda7bbb8ff18cf9e11cfdf8e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a86ff5fbeda7bbb8ff18cf9e11cfdf8e columnFamilyName testFamily 2023-10-04T08:14:38,726 DEBUG [StoreOpener-a86ff5fbeda7bbb8ff18cf9e11cfdf8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:38,726 INFO [StoreOpener-a86ff5fbeda7bbb8ff18cf9e11cfdf8e-1 {}] regionserver.HStore(324): Store=a86ff5fbeda7bbb8ff18cf9e11cfdf8e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:38,727 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,728 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,730 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1076): writing seq id for a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:14:38,732 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:38,733 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1093): Opened a86ff5fbeda7bbb8ff18cf9e11cfdf8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70602268, jitterRate=0.05205577611923218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:38,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(992): Region open journal for a86ff5fbeda7bbb8ff18cf9e11cfdf8e: 2023-10-04T08:14:38,734 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., pid=86, masterSystemTime=1696407278718 2023-10-04T08:14:38,735 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,735 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:14:38,736 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:38,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=86, resume processing ppid=85 2023-10-04T08:14:38,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure a86ff5fbeda7bbb8ff18cf9e11cfdf8e, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-10-04T08:14:38,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=a86ff5fbeda7bbb8ff18cf9e11cfdf8e, ASSIGN in 327 msec 2023-10-04T08:14:38,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:38,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407278740"}]},"ts":"1696407278740"} 2023-10-04T08:14:38,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:38,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:38,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 772 msec 2023-10-04T08:14:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=84 2023-10-04T08:14:38,977 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2023-10-04T08:14:38,977 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:38,977 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:38,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2023-10-04T08:14:38,981 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885273 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:38,982 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:38,982 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:38,983 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2023-10-04T08:14:38,985 DEBUG [hconnection-0x638dc38c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:38,987 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35934, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:38,993 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:38,995 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:14:38,996 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:14:38,997 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x5f8b1bc7 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:14:38,997 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x5f8b1bc7 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:38,998 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35942, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:14:39,001 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:14:41,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2023-10-04T08:14:41,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:41,829 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666391 expired clientIPAndPort=172.31.12.81:44470, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:14:44,225 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:44,387 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2023-10-04T08:14:45,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2023-10-04T08:14:45,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2023-10-04T08:14:45,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:14:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 89 bytes 2023-10-04T08:14:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:14:46,023 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2023-10-04T08:14:47,934 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666386 expired clientIPAndPort=172.31.12.81:37854, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:14:47,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1973ms 2023-10-04T08:14:47,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1973ms 2023-10-04T08:14:47,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:14:48,000 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-10-04T08:14:45.010Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6009ms, rpcTimeout=6000ms 2023-10-04T08:14:47.999Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1973ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1977ms, rpcTimeout=1973ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-10-04T08:14:48,002 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2023-10-04T08:14:48,002 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x224eba2a to 127.0.0.1:50391 2023-10-04T08:14:48,002 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:48,002 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:14:48,002 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:14:48,003 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:14:48,003 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5f8b1bc7 to 127.0.0.1:50391 2023-10-04T08:14:48,014 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=214 (was 214), OpenFileDescriptor=482 (was 481) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=118 (was 121), ProcessCount=167 (was 167), AvailableMemoryMB=6012 (was 6029) 2023-10-04T08:14:48,024 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=214, OpenFileDescriptor=482, MaxFileDescriptor=60000, SystemLoadAverage=118, ProcessCount=167, AvailableMemoryMB=6011 2023-10-04T08:14:48,025 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7dc48400 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:48,029 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x7dc48400 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f734459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:48,029 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x56088d11 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:14:48,034 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55787479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:14:48,036 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:14:48,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2023-10-04T08:14:48,038 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:14:48,038 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:48,038 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2023-10-04T08:14:48,038 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:14:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-04T08:14:48,450 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => abec73fca0361fedcb0360a78d99052d, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing abec73fca0361fedcb0360a78d99052d, disabling compactions & flushes 2023-10-04T08:14:48,859 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. after waiting 0 ms 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:48,859 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:48,859 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for abec73fca0361fedcb0360a78d99052d: 2023-10-04T08:14:48,860 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:14:48,861 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1696407288860"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407288860"}]},"ts":"1696407288860"} 2023-10-04T08:14:48,862 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:14:48,863 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:14:48,863 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407288863"}]},"ts":"1696407288863"} 2023-10-04T08:14:48,864 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:14:48,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=abec73fca0361fedcb0360a78d99052d, ASSIGN}] 2023-10-04T08:14:48,869 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=abec73fca0361fedcb0360a78d99052d, ASSIGN 2023-10-04T08:14:48,870 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=abec73fca0361fedcb0360a78d99052d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:14:49,021 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=abec73fca0361fedcb0360a78d99052d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:49,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure abec73fca0361fedcb0360a78d99052d, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:14:49,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-04T08:14:49,179 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:49,179 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7254): Opening region: {ENCODED => abec73fca0361fedcb0360a78d99052d, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:14:49,180 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,180 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:14:49,180 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7296): checking encryption for abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,180 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7299): checking classloading for abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,181 INFO [StoreOpener-abec73fca0361fedcb0360a78d99052d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,183 INFO [StoreOpener-abec73fca0361fedcb0360a78d99052d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abec73fca0361fedcb0360a78d99052d columnFamilyName testFamily 2023-10-04T08:14:49,183 DEBUG [StoreOpener-abec73fca0361fedcb0360a78d99052d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:14:49,184 INFO [StoreOpener-abec73fca0361fedcb0360a78d99052d-1 {}] regionserver.HStore(324): Store=abec73fca0361fedcb0360a78d99052d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:14:49,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,185 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1076): writing seq id for abec73fca0361fedcb0360a78d99052d 2023-10-04T08:14:49,189 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:14:49,189 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1093): Opened abec73fca0361fedcb0360a78d99052d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71003088, jitterRate=0.058028459548950195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:14:49,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(992): Region open journal for abec73fca0361fedcb0360a78d99052d: 2023-10-04T08:14:49,191 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d., pid=89, masterSystemTime=1696407289174 2023-10-04T08:14:49,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:49,192 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:14:49,193 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=abec73fca0361fedcb0360a78d99052d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:14:49,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=89, resume processing ppid=88 2023-10-04T08:14:49,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure abec73fca0361fedcb0360a78d99052d, server=jenkins-hbase3.apache.org,45997,1696407041329 in 171 msec 2023-10-04T08:14:49,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-10-04T08:14:49,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=abec73fca0361fedcb0360a78d99052d, ASSIGN in 327 msec 2023-10-04T08:14:49,197 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:14:49,197 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407289197"}]},"ts":"1696407289197"} 2023-10-04T08:14:49,198 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:14:49,200 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:14:49,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1640 sec 2023-10-04T08:14:50,388 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2023-10-04T08:14:50,436 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666379 expired clientIPAndPort=172.31.12.81:53346, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:14:50,436 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666380 expired clientIPAndPort=172.31.12.81:53346, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:14:50,436 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666382 expired clientIPAndPort=172.31.12.81:53346, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:14:51,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2023-10-04T08:14:51,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:14:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-04T08:14:51,042 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2023-10-04T08:14:51,042 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:14:51,042 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:51,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32861 bytes) of info 2023-10-04T08:14:51,046 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885280 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:51,047 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2023-10-04T08:14:51,047 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:14:51,047 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2023-10-04T08:14:51,048 DEBUG [hconnection-0x32553982-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:14:51,050 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55198, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:14:51,056 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:14:51,058 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2023-10-04T08:14:51,059 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2023-10-04T08:14:51,060 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885283 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:51,060 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2023-10-04T08:14:51,062 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885283 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:14:51,062 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2023-10-04T08:14:52,523 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:14:54,681 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:14:55,540 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666373 expired clientIPAndPort=172.31.12.81:53360, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:00,072 DEBUG [Listener at localhost/43533 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10016: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9007ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10016: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9007ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9007ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9007ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2023-10-04T08:15:00,072 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9011 2023-10-04T08:15:00,072 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x56088d11 to 127.0.0.1:50391 2023-10-04T08:15:00,072 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:00,073 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:15:00,073 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:15:00,073 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:00,073 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7dc48400 to 127.0.0.1:50391 2023-10-04T08:15:00,084 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=212 (was 214), OpenFileDescriptor=478 (was 482), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=114 (was 118), ProcessCount=170 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=5992 (was 6011) 2023-10-04T08:15:00,093 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=212, OpenFileDescriptor=478, MaxFileDescriptor=60000, SystemLoadAverage=114, ProcessCount=170, AvailableMemoryMB=5992 2023-10-04T08:15:00,094 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x44de88b2 to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:15:00,099 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x44de88b2 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf23a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:15:00,099 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x74f5b7db to 127.0.0.1:50391 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-04T08:15:00,104 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c29b369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2023-10-04T08:15:00,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-04T08:15:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] procedure2.ProcedureExecutor(1032): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2023-10-04T08:15:00,108 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2023-10-04T08:15:00,108 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:15:00,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2023-10-04T08:15:00,109 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-04T08:15:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-10-04T08:15:00,143 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666368 expired clientIPAndPort=172.31.12.81:53362, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:00,522 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0eeaa61c399a56345cddfcafd2aa53ca, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62 2023-10-04T08:15:00,563 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 0eeaa61c399a56345cddfcafd2aa53ca, disabling compactions & flushes 2023-10-04T08:15:00,931 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. after waiting 0 ms 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:00,931 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:00,931 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0eeaa61c399a56345cddfcafd2aa53ca: 2023-10-04T08:15:00,932 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2023-10-04T08:15:00,933 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1696407300933"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696407300933"}]},"ts":"1696407300933"} 2023-10-04T08:15:00,934 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-04T08:15:00,935 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-04T08:15:00,935 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407300935"}]},"ts":"1696407300935"} 2023-10-04T08:15:00,936 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2023-10-04T08:15:00,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0eeaa61c399a56345cddfcafd2aa53ca, ASSIGN}] 2023-10-04T08:15:00,942 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0eeaa61c399a56345cddfcafd2aa53ca, ASSIGN 2023-10-04T08:15:00,943 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0eeaa61c399a56345cddfcafd2aa53ca, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,45997,1696407041329; forceNewPlan=false, retain=false 2023-10-04T08:15:01,094 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=0eeaa61c399a56345cddfcafd2aa53ca, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:15:01,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 0eeaa61c399a56345cddfcafd2aa53ca, server=jenkins-hbase3.apache.org,45997,1696407041329}] 2023-10-04T08:15:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-10-04T08:15:01,254 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:01,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7254): Opening region: {ENCODED => 0eeaa61c399a56345cddfcafd2aa53ca, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca.', STARTKEY => '', ENDKEY => ''} 2023-10-04T08:15:01,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-04T08:15:01,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7296): checking encryption for 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,254 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7299): checking classloading for 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,256 INFO [StoreOpener-0eeaa61c399a56345cddfcafd2aa53ca-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,257 INFO [StoreOpener-0eeaa61c399a56345cddfcafd2aa53ca-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eeaa61c399a56345cddfcafd2aa53ca columnFamilyName testFamily 2023-10-04T08:15:01,257 DEBUG [StoreOpener-0eeaa61c399a56345cddfcafd2aa53ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-04T08:15:01,258 INFO [StoreOpener-0eeaa61c399a56345cddfcafd2aa53ca-1 {}] regionserver.HStore(324): Store=0eeaa61c399a56345cddfcafd2aa53ca/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-04T08:15:01,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1076): writing seq id for 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:01,264 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-04T08:15:01,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): Opened 0eeaa61c399a56345cddfcafd2aa53ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61596793, jitterRate=-0.08213625848293304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-04T08:15:01,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(992): Region open journal for 0eeaa61c399a56345cddfcafd2aa53ca: 2023-10-04T08:15:01,266 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2556): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., pid=92, masterSystemTime=1696407301247 2023-10-04T08:15:01,267 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2583): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:01,268 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:01,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=0eeaa61c399a56345cddfcafd2aa53ca, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:15:01,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-10-04T08:15:01,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 0eeaa61c399a56345cddfcafd2aa53ca, server=jenkins-hbase3.apache.org,45997,1696407041329 in 174 msec 2023-10-04T08:15:01,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=91, resume processing ppid=90 2023-10-04T08:15:01,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=0eeaa61c399a56345cddfcafd2aa53ca, ASSIGN in 330 msec 2023-10-04T08:15:01,273 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-04T08:15:01,273 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696407301273"}]},"ts":"1696407301273"} 2023-10-04T08:15:01,274 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2023-10-04T08:15:01,278 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2023-10-04T08:15:01,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1720 sec 2023-10-04T08:15:02,145 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666366 expired clientIPAndPort=172.31.12.81:53362, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:02,390 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2023-10-04T08:15:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46181 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=90 2023-10-04T08:15:03,112 INFO [Listener at localhost/43533 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2023-10-04T08:15:03,112 DEBUG [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2023-10-04T08:15:03,113 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:15:03,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32921 bytes) of info 2023-10-04T08:15:03,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 17993814749866885284 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2023-10-04T08:15:03,118 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2023-10-04T08:15:03,118 INFO [Listener at localhost/43533 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-04T08:15:03,118 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2023-10-04T08:15:03,126 DEBUG [hconnection-0x5fc65e1f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:15:03,128 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:15:03,136 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2023-10-04T08:15:03,138 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2023-10-04T08:15:03,138 DEBUG [Listener at localhost/43533 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2023-10-04T08:15:03,139 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x44de88b2 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=-1] 2023-10-04T08:15:03,140 DEBUG [ReadOnlyZKClient-127.0.0.1:50391@0x44de88b2 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-04T08:15:03,142 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:39152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-04T08:15:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:15:03,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:15:04,149 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2023-10-04T08:15:04,149 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2023-10-04T08:15:04,150 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:15:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(191): Unknown callId: 1, skipping over this response of 89 bytes 2023-10-04T08:15:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2] 2023-10-04T08:15:05,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2023-10-04T08:15:06,139 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=967ms, rpcTimeout=965ms 2023-10-04T08:15:06,139 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=967ms, rpcTimeout=965ms 2023-10-04T08:15:06,139 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca., hostname=jenkins-hbase3.apache.org,45997,1696407041329, seqNum=2 from cache 2023-10-04T08:15:06,140 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-10-04T08:15:04.150Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2023-10-04T08:15:06.139Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=967ms, rpcTimeout=965ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.6.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.4.jar:4.1.4] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=967ms, rpcTimeout=965ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=967ms, rpcTimeout=965ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 6 more 2023-10-04T08:15:06,143 INFO [Listener at localhost/43533 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2023-10-04T08:15:06,143 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x74f5b7db to 127.0.0.1:50391 2023-10-04T08:15:06,143 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:06,143 INFO [Listener at localhost/43533 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/43533. 2023-10-04T08:15:06,143 DEBUG [Listener at localhost/43533 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2023-10-04T08:15:06,143 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:06,144 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x44de88b2 to 127.0.0.1:50391 2023-10-04T08:15:06,156 INFO [Listener at localhost/43533 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=214 (was 212) - Thread LEAK? -, OpenFileDescriptor=482 (was 478) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=113 (was 114), ProcessCount=170 (was 170), AvailableMemoryMB=5986 (was 5992) 2023-10-04T08:15:06,156 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-04T08:15:06,156 INFO [Listener at localhost/43533 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-04T08:15:06,156 DEBUG [Listener at localhost/43533 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x47dbb7dd to 127.0.0.1:50391 2023-10-04T08:15:06,156 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:06,156 DEBUG [Listener at localhost/43533 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-04T08:15:06,156 DEBUG [Listener at localhost/43533 {}] util.JVMClusterUtil(257): Found active master hash=1119900838, stopped=false 2023-10-04T08:15:06,157 INFO [Listener at localhost/43533 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:15:06,159 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-10-04T08:15:06,159 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2023-10-04T08:15:06,159 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:15:06,159 INFO [Listener at localhost/43533 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-04T08:15:06,159 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:15:06,159 DEBUG [Listener at localhost/43533 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:06,159 INFO [Listener at localhost/43533 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,45997,1696407041329' ***** 2023-10-04T08:15:06,159 INFO [Listener at localhost/43533 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-04T08:15:06,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-10-04T08:15:06,160 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-04T08:15:06,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2023-10-04T08:15:06,160 INFO [RS:0;jenkins-hbase3:45997 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-04T08:15:06,160 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-04T08:15:06,160 INFO [RS:0;jenkins-hbase3:45997 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-04T08:15:06,160 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 999353f1059233c76a4049e6624f9aa5 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 7280776114ac4274bfa1c2878b754e7e 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for abec73fca0361fedcb0360a78d99052d 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 9fe8cf8cd62090355177a9f2de04a723 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 4958717ea3998bbee14b70ccbd8d2b06 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 71b5c7d5f3c2000b3e5dbfd8bb238abb 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for aa20ab650b256298cc3fd6d02def3716 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for daa135c3e71fa3e0d3b0ca3000158a02 2023-10-04T08:15:06,161 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for b1b68c5387402f2415aa871d52ff711f 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 1bcae92c528d1fed7f31bf8de10093d1 2023-10-04T08:15:06,161 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 999353f1059233c76a4049e6624f9aa5, disabling compactions & flushes 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 015af7fd5d2067013019f6874c98f002 2023-10-04T08:15:06,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 17f6bc2dc04de9b433ce6efaae857bb1 2023-10-04T08:15:06,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:15:06,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. after waiting 0 ms 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 7846ddcf7f67173b45f510faad9a2446 2023-10-04T08:15:06,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 051899d47523a890d1ad457722c3b97f 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 87fdd579d0b26040c8abef46e9d1ae20 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 9c208f35f121b8b6fe14f4139d19c72e 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 59fbb4b736d248cd85fe604c5ee58655 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 0eeaa61c399a56345cddfcafd2aa53ca 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 6438062196d04176b2d1947e41c0d3e6 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 8b44dd21679fb8b47ec68172b7a9d6ab 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for a86ff5fbeda7bbb8ff18cf9e11cfdf8e 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 96dd4b21100c44df0f1d2c389effacfe 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 7cf4420553a66064d10b73e18ff7ce65 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 87c60d1bc9734410abc9413183bad22a 2023-10-04T08:15:06,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 999353f1059233c76a4049e6624f9aa5 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 2b8b1fed447fcf6ffb03157a004d4e73 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 3443a3e340fdb17279ac1f79e8b74717 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:06,162 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:15:06,163 DEBUG [RS:0;jenkins-hbase3:45997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:06,163 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-04T08:15:06,163 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-04T08:15:06,163 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-04T08:15:06,163 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-04T08:15:06,163 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 30 regions to close 2023-10-04T08:15:06,163 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-04T08:15:06,163 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-04T08:15:06,164 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-04T08:15:06,164 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-04T08:15:06,164 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-04T08:15:06,164 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=48.69 KB heapSize=75.08 KB 2023-10-04T08:15:06,165 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 1588230740, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7280776114ac4274bfa1c2878b754e7e, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 999353f1059233c76a4049e6624f9aa5, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,375 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 1588230740, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7280776114ac4274bfa1c2878b754e7e, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 999353f1059233c76a4049e6624f9aa5, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,427 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/.tmp/testFamily/82276665c119483cb10603444a4dd222 is 61, key is row-0/testFamily:testQualifier/1696407147502/Put/seqid=0 2023-10-04T08:15:06,433 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/info/4de8a7cefb89466d98043e5c922f68f6 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e./info:regioninfo/1696407106048/Put/seqid=0 2023-10-04T08:15:06,575 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 1588230740, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7280776114ac4274bfa1c2878b754e7e, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 999353f1059233c76a4049e6624f9aa5, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,673 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.4.jar:?] 2023-10-04T08:15:06,776 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 1588230740, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7280776114ac4274bfa1c2878b754e7e, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 999353f1059233c76a4049e6624f9aa5, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,847 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/.tmp/testFamily/82276665c119483cb10603444a4dd222 2023-10-04T08:15:06,847 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.45 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/info/4de8a7cefb89466d98043e5c922f68f6 2023-10-04T08:15:06,903 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/.tmp/testFamily/82276665c119483cb10603444a4dd222 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/testFamily/82276665c119483cb10603444a4dd222 2023-10-04T08:15:06,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/testFamily/82276665c119483cb10603444a4dd222, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:06,913 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 999353f1059233c76a4049e6624f9aa5 in 751ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:06,917 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/table/8aaca50364d7407492173f99ea09de63 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1696407106059/Put/seqid=0 2023-10-04T08:15:06,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_0_-false/999353f1059233c76a4049e6624f9aa5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:06,923 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:15:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 999353f1059233c76a4049e6624f9aa5: 2023-10-04T08:15:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1696407144479.999353f1059233c76a4049e6624f9aa5. 2023-10-04T08:15:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7280776114ac4274bfa1c2878b754e7e, disabling compactions & flushes 2023-10-04T08:15:06,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:15:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:15:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. after waiting 0 ms 2023-10-04T08:15:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:15:06,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7280776114ac4274bfa1c2878b754e7e 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:06,925 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/table/8aaca50364d7407492173f99ea09de63 2023-10-04T08:15:06,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:15:06,937 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/info/4de8a7cefb89466d98043e5c922f68f6 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/info/4de8a7cefb89466d98043e5c922f68f6 2023-10-04T08:15:06,940 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/.tmp/testFamily/1f4d47d98732472d8bc5630807a3c69f is 61, key is row-0/testFamily:testQualifier/1696407111064/Put/seqid=0 2023-10-04T08:15:06,944 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/info/4de8a7cefb89466d98043e5c922f68f6, entries=290, sequenceid=149, filesize=51.4 K 2023-10-04T08:15:06,949 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/.tmp/table/8aaca50364d7407492173f99ea09de63 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/table/8aaca50364d7407492173f99ea09de63 2023-10-04T08:15:06,950 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/.tmp/testFamily/1f4d47d98732472d8bc5630807a3c69f 2023-10-04T08:15:06,955 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-04T08:15:06,955 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-04T08:15:06,956 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/table/8aaca50364d7407492173f99ea09de63, entries=58, sequenceid=149, filesize=9.6 K 2023-10-04T08:15:06,957 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~48.69 KB/49854, heapSize ~74.80 KB/76592, currentSize=0 B/0 for 1588230740 in 793ms, sequenceid=149, compaction requested=false 2023-10-04T08:15:06,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/.tmp/testFamily/1f4d47d98732472d8bc5630807a3c69f as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/testFamily/1f4d47d98732472d8bc5630807a3c69f 2023-10-04T08:15:06,965 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/testFamily/1f4d47d98732472d8bc5630807a3c69f, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:06,967 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7280776114ac4274bfa1c2878b754e7e in 42ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:06,969 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2023-10-04T08:15:06,970 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-04T08:15:06,971 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-04T08:15:06,971 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-04T08:15:06,971 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-04T08:15:06,973 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/7280776114ac4274bfa1c2878b754e7e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:06,974 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:15:06,974 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7280776114ac4274bfa1c2878b754e7e: 2023-10-04T08:15:06,974 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1696407108037.7280776114ac4274bfa1c2878b754e7e. 2023-10-04T08:15:06,974 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing abec73fca0361fedcb0360a78d99052d, disabling compactions & flushes 2023-10-04T08:15:06,975 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:15:06,975 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:15:06,975 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. after waiting 0 ms 2023-10-04T08:15:06,975 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:15:06,975 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing abec73fca0361fedcb0360a78d99052d 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:06,976 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:06,988 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/.tmp/testFamily/ff6af14b00cc485bb5521e06e903522e is 61, key is row-0/testFamily:testQualifier/1696407291052/Put/seqid=0 2023-10-04T08:15:07,176 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 27 regions to close 2023-10-04T08:15:07,176 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:07,377 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, abec73fca0361fedcb0360a78d99052d, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:07,398 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/.tmp/testFamily/ff6af14b00cc485bb5521e06e903522e 2023-10-04T08:15:07,406 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/.tmp/testFamily/ff6af14b00cc485bb5521e06e903522e as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/testFamily/ff6af14b00cc485bb5521e06e903522e 2023-10-04T08:15:07,412 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/testFamily/ff6af14b00cc485bb5521e06e903522e, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:07,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for abec73fca0361fedcb0360a78d99052d in 441ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:07,421 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_1_-true/abec73fca0361fedcb0360a78d99052d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:07,422 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:15:07,422 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for abec73fca0361fedcb0360a78d99052d: 2023-10-04T08:15:07,422 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1696407288035.abec73fca0361fedcb0360a78d99052d. 2023-10-04T08:15:07,422 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9fe8cf8cd62090355177a9f2de04a723, disabling compactions & flushes 2023-10-04T08:15:07,422 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:15:07,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:15:07,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. after waiting 0 ms 2023-10-04T08:15:07,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:15:07,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9fe8cf8cd62090355177a9f2de04a723 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:07,435 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/.tmp/testFamily/6dcc78a007dd4234a40a967ca87f82b0 is 61, key is row-0/testFamily:testQualifier/1696407260274/Put/seqid=0 2023-10-04T08:15:07,577 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:07,777 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, 9fe8cf8cd62090355177a9f2de04a723, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:07,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/.tmp/testFamily/6dcc78a007dd4234a40a967ca87f82b0 2023-10-04T08:15:07,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/.tmp/testFamily/6dcc78a007dd4234a40a967ca87f82b0 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/testFamily/6dcc78a007dd4234a40a967ca87f82b0 2023-10-04T08:15:07,857 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/testFamily/6dcc78a007dd4234a40a967ca87f82b0, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:07,859 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9fe8cf8cd62090355177a9f2de04a723 in 436ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:07,864 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_1_-true/9fe8cf8cd62090355177a9f2de04a723/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:07,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9fe8cf8cd62090355177a9f2de04a723: 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1696407257255.9fe8cf8cd62090355177a9f2de04a723. 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4958717ea3998bbee14b70ccbd8d2b06, disabling compactions & flushes 2023-10-04T08:15:07,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. after waiting 0 ms 2023-10-04T08:15:07,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:15:07,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 4958717ea3998bbee14b70ccbd8d2b06 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:07,879 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/.tmp/testFamily/58a15ca97f7c4e57acc5e82f8c201470 is 61, key is row-0/testFamily:testQualifier/1696407270909/Put/seqid=0 2023-10-04T08:15:07,978 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:08,176 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(534): Scanner lease -452929323842666361 expired clientIPAndPort=172.31.12.81:37544, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:08,178 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 25 regions to close 2023-10-04T08:15:08,178 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 4958717ea3998bbee14b70ccbd8d2b06, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 71b5c7d5f3c2000b3e5dbfd8bb238abb, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:08,286 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/.tmp/testFamily/58a15ca97f7c4e57acc5e82f8c201470 2023-10-04T08:15:08,294 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/.tmp/testFamily/58a15ca97f7c4e57acc5e82f8c201470 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/testFamily/58a15ca97f7c4e57acc5e82f8c201470 2023-10-04T08:15:08,300 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/testFamily/58a15ca97f7c4e57acc5e82f8c201470, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:08,302 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4958717ea3998bbee14b70ccbd8d2b06 in 437ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:08,307 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/4958717ea3998bbee14b70ccbd8d2b06/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:08,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4958717ea3998bbee14b70ccbd8d2b06: 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1696407267889.4958717ea3998bbee14b70ccbd8d2b06. 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 71b5c7d5f3c2000b3e5dbfd8bb238abb, disabling compactions & flushes 2023-10-04T08:15:08,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. after waiting 0 ms 2023-10-04T08:15:08,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:15:08,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 71b5c7d5f3c2000b3e5dbfd8bb238abb 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:08,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/.tmp/testFamily/7d07d04a941e4f69a2d69cdba5734e08 is 61, key is row-0/testFamily:testQualifier/1696407177928/Put/seqid=0 2023-10-04T08:15:08,329 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/.tmp/testFamily/7d07d04a941e4f69a2d69cdba5734e08 2023-10-04T08:15:08,336 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/.tmp/testFamily/7d07d04a941e4f69a2d69cdba5734e08 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/testFamily/7d07d04a941e4f69a2d69cdba5734e08 2023-10-04T08:15:08,341 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/testFamily/7d07d04a941e4f69a2d69cdba5734e08, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:08,342 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 71b5c7d5f3c2000b3e5dbfd8bb238abb in 34ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:08,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/71b5c7d5f3c2000b3e5dbfd8bb238abb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:08,350 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 71b5c7d5f3c2000b3e5dbfd8bb238abb: 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1696407174908.71b5c7d5f3c2000b3e5dbfd8bb238abb. 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing aa20ab650b256298cc3fd6d02def3716, disabling compactions & flushes 2023-10-04T08:15:08,350 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. after waiting 0 ms 2023-10-04T08:15:08,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:15:08,350 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing aa20ab650b256298cc3fd6d02def3716 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:08,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/.tmp/testFamily/8ef55fc9557948abbcef72e619ef342a is 61, key is row-0/testFamily:testQualifier/1696407161165/Put/seqid=0 2023-10-04T08:15:08,378 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:08,578 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:08,769 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/.tmp/testFamily/8ef55fc9557948abbcef72e619ef342a 2023-10-04T08:15:08,776 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/.tmp/testFamily/8ef55fc9557948abbcef72e619ef342a as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/testFamily/8ef55fc9557948abbcef72e619ef342a 2023-10-04T08:15:08,779 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, aa20ab650b256298cc3fd6d02def3716, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:08,783 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/testFamily/8ef55fc9557948abbcef72e619ef342a, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:08,784 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aa20ab650b256298cc3fd6d02def3716 in 434ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:08,789 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnClose_0_-false/aa20ab650b256298cc3fd6d02def3716/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:08,790 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for aa20ab650b256298cc3fd6d02def3716: 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1696407158140.aa20ab650b256298cc3fd6d02def3716. 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing daa135c3e71fa3e0d3b0ca3000158a02, disabling compactions & flushes 2023-10-04T08:15:08,790 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. after waiting 0 ms 2023-10-04T08:15:08,790 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:15:08,790 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing daa135c3e71fa3e0d3b0ca3000158a02 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:08,803 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/.tmp/testFamily/de75399bf8d04a6d899bc9d2d554c71b is 61, key is row-0/testFamily:testQualifier/1696407190000/Put/seqid=0 2023-10-04T08:15:08,979 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:09,179 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 22 regions to close 2023-10-04T08:15:09,179 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, daa135c3e71fa3e0d3b0ca3000158a02, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:09,214 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/.tmp/testFamily/de75399bf8d04a6d899bc9d2d554c71b 2023-10-04T08:15:09,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/.tmp/testFamily/de75399bf8d04a6d899bc9d2d554c71b as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/testFamily/de75399bf8d04a6d899bc9d2d554c71b 2023-10-04T08:15:09,228 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/testFamily/de75399bf8d04a6d899bc9d2d554c71b, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:09,229 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for daa135c3e71fa3e0d3b0ca3000158a02 in 439ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:09,234 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNext_0_-false/daa135c3e71fa3e0d3b0ca3000158a02/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:09,235 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:15:09,235 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for daa135c3e71fa3e0d3b0ca3000158a02: 2023-10-04T08:15:09,235 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1696407186978.daa135c3e71fa3e0d3b0ca3000158a02. 2023-10-04T08:15:09,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b1b68c5387402f2415aa871d52ff711f, disabling compactions & flushes 2023-10-04T08:15:09,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:15:09,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:15:09,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. after waiting 0 ms 2023-10-04T08:15:09,236 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:15:09,236 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b1b68c5387402f2415aa871d52ff711f 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:09,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/.tmp/testFamily/a17e54c37b724f55a279c7a1d8f51b63 is 61, key is row-0/testFamily:testQualifier/1696407136335/Put/seqid=0 2023-10-04T08:15:09,380 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:09,580 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b1b68c5387402f2415aa871d52ff711f, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:09,655 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/.tmp/testFamily/a17e54c37b724f55a279c7a1d8f51b63 2023-10-04T08:15:09,662 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/.tmp/testFamily/a17e54c37b724f55a279c7a1d8f51b63 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/testFamily/a17e54c37b724f55a279c7a1d8f51b63 2023-10-04T08:15:09,668 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/testFamily/a17e54c37b724f55a279c7a1d8f51b63, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:09,669 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b1b68c5387402f2415aa871d52ff711f in 433ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:09,674 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_0_-false/b1b68c5387402f2415aa871d52ff711f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:09,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b1b68c5387402f2415aa871d52ff711f: 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1696407133313.b1b68c5387402f2415aa871d52ff711f. 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1bcae92c528d1fed7f31bf8de10093d1, disabling compactions & flushes 2023-10-04T08:15:09,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. after waiting 0 ms 2023-10-04T08:15:09,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:09,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 1bcae92c528d1fed7f31bf8de10093d1 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:09,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/.tmp/testFamily/d6427ee8b07449f98f0256a03606e3c7 is 61, key is row-0/testFamily:testQualifier/1696407238563/Put/seqid=0 2023-10-04T08:15:09,780 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:09,980 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 1bcae92c528d1fed7f31bf8de10093d1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,094 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/.tmp/testFamily/d6427ee8b07449f98f0256a03606e3c7 2023-10-04T08:15:10,101 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/.tmp/testFamily/d6427ee8b07449f98f0256a03606e3c7 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/testFamily/d6427ee8b07449f98f0256a03606e3c7 2023-10-04T08:15:10,107 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/testFamily/d6427ee8b07449f98f0256a03606e3c7, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:10,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1bcae92c528d1fed7f31bf8de10093d1 in 433ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:10,113 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/1bcae92c528d1fed7f31bf8de10093d1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:10,114 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:10,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1bcae92c528d1fed7f31bf8de10093d1: 2023-10-04T08:15:10,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1696407235546.1bcae92c528d1fed7f31bf8de10093d1. 2023-10-04T08:15:10,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 015af7fd5d2067013019f6874c98f002, disabling compactions & flushes 2023-10-04T08:15:10,114 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:15:10,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:15:10,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. after waiting 0 ms 2023-10-04T08:15:10,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:15:10,115 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 015af7fd5d2067013019f6874c98f002 1/1 column families, dataSize=78 B heapSize=488 B 2023-10-04T08:15:10,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/.tmp/info/fb24c1a4f63546529d57f2ad23e1cd5c is 45, key is default/info:d/1696407104588/Put/seqid=0 2023-10-04T08:15:10,181 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 19 regions to close 2023-10-04T08:15:10,181 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,381 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 015af7fd5d2067013019f6874c98f002, 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,540 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/.tmp/info/fb24c1a4f63546529d57f2ad23e1cd5c 2023-10-04T08:15:10,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/.tmp/info/fb24c1a4f63546529d57f2ad23e1cd5c as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/info/fb24c1a4f63546529d57f2ad23e1cd5c 2023-10-04T08:15:10,553 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/info/fb24c1a4f63546529d57f2ad23e1cd5c, entries=2, sequenceid=6, filesize=4.9 K 2023-10-04T08:15:10,554 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 015af7fd5d2067013019f6874c98f002 in 439ms, sequenceid=6, compaction requested=false 2023-10-04T08:15:10,560 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/namespace/015af7fd5d2067013019f6874c98f002/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-10-04T08:15:10,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 015af7fd5d2067013019f6874c98f002: 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696407103615.015af7fd5d2067013019f6874c98f002. 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 17f6bc2dc04de9b433ce6efaae857bb1, disabling compactions & flushes 2023-10-04T08:15:10,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. after waiting 0 ms 2023-10-04T08:15:10,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:15:10,561 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 17f6bc2dc04de9b433ce6efaae857bb1 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:10,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/.tmp/testFamily/59cb17666a734edf86755fea10f64082 is 61, key is row-0/testFamily:testQualifier/1696407264834/Put/seqid=0 2023-10-04T08:15:10,581 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,782 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/.tmp/testFamily/59cb17666a734edf86755fea10f64082 2023-10-04T08:15:10,982 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 17f6bc2dc04de9b433ce6efaae857bb1, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:10,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/.tmp/testFamily/59cb17666a734edf86755fea10f64082 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/testFamily/59cb17666a734edf86755fea10f64082 2023-10-04T08:15:10,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/testFamily/59cb17666a734edf86755fea10f64082, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:10,994 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17f6bc2dc04de9b433ce6efaae857bb1 in 433ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:10,999 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_1_-true/17f6bc2dc04de9b433ce6efaae857bb1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:11,000 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:15:11,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 17f6bc2dc04de9b433ce6efaae857bb1: 2023-10-04T08:15:11,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1696407261815.17f6bc2dc04de9b433ce6efaae857bb1. 2023-10-04T08:15:11,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7846ddcf7f67173b45f510faad9a2446, disabling compactions & flushes 2023-10-04T08:15:11,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:15:11,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:15:11,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. after waiting 0 ms 2023-10-04T08:15:11,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:15:11,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7846ddcf7f67173b45f510faad9a2446 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:11,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/.tmp/testFamily/5ad30d27670f457ab7fdad508e4357bb is 61, key is row-0/testFamily:testQualifier/1696407205174/Put/seqid=0 2023-10-04T08:15:11,029 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2023-10-04T08:15:11,029 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-04T08:15:11,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2023-10-04T08:15:11,031 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2023-10-04T08:15:11,031 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2023-10-04T08:15:11,182 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 17 regions to close 2023-10-04T08:15:11,182 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:11,382 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7846ddcf7f67173b45f510faad9a2446, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:11,424 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/.tmp/testFamily/5ad30d27670f457ab7fdad508e4357bb 2023-10-04T08:15:11,432 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/.tmp/testFamily/5ad30d27670f457ab7fdad508e4357bb as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/testFamily/5ad30d27670f457ab7fdad508e4357bb 2023-10-04T08:15:11,437 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/testFamily/5ad30d27670f457ab7fdad508e4357bb, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:11,438 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7846ddcf7f67173b45f510faad9a2446 in 437ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:11,443 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/7846ddcf7f67173b45f510faad9a2446/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:11,444 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:15:11,444 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7846ddcf7f67173b45f510faad9a2446: 2023-10-04T08:15:11,444 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1696407202155.7846ddcf7f67173b45f510faad9a2446. 2023-10-04T08:15:11,444 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fdbb6b7904791beef7b9e29a91436269, disabling compactions & flushes 2023-10-04T08:15:11,444 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:15:11,444 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:15:11,445 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. after waiting 0 ms 2023-10-04T08:15:11,445 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:15:11,445 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing fdbb6b7904791beef7b9e29a91436269 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:11,457 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/.tmp/testFamily/e4491bcd11814d8b881954e105a86685 is 61, key is row-0/testFamily:testQualifier/1696407199090/Put/seqid=0 2023-10-04T08:15:11,583 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:11,783 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a, fdbb6b7904791beef7b9e29a91436269 2023-10-04T08:15:11,864 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/.tmp/testFamily/e4491bcd11814d8b881954e105a86685 2023-10-04T08:15:11,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/.tmp/testFamily/e4491bcd11814d8b881954e105a86685 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/testFamily/e4491bcd11814d8b881954e105a86685 2023-10-04T08:15:11,876 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/testFamily/e4491bcd11814d8b881954e105a86685, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:11,877 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fdbb6b7904791beef7b9e29a91436269 in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:11,882 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/fdbb6b7904791beef7b9e29a91436269/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:11,883 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fdbb6b7904791beef7b9e29a91436269: 2023-10-04T08:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1696407196071.fdbb6b7904791beef7b9e29a91436269. 2023-10-04T08:15:11,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 051899d47523a890d1ad457722c3b97f, disabling compactions & flushes 2023-10-04T08:15:11,884 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. after waiting 0 ms 2023-10-04T08:15:11,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:15:11,884 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 051899d47523a890d1ad457722c3b97f 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:11,896 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/.tmp/testFamily/35e98f39d5f84f28928fea4e8eb67bee is 61, key is row-0/testFamily:testQualifier/1696407133242/Put/seqid=0 2023-10-04T08:15:11,983 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:12,183 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 15 regions to close 2023-10-04T08:15:12,184 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 051899d47523a890d1ad457722c3b97f, 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:12,303 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/.tmp/testFamily/35e98f39d5f84f28928fea4e8eb67bee 2023-10-04T08:15:12,310 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/.tmp/testFamily/35e98f39d5f84f28928fea4e8eb67bee as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/testFamily/35e98f39d5f84f28928fea4e8eb67bee 2023-10-04T08:15:12,315 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/testFamily/35e98f39d5f84f28928fea4e8eb67bee, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:12,316 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 051899d47523a890d1ad457722c3b97f in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:12,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_0_-false/051899d47523a890d1ad457722c3b97f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:12,325 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:15:12,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 051899d47523a890d1ad457722c3b97f: 2023-10-04T08:15:12,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1696407130219.051899d47523a890d1ad457722c3b97f. 2023-10-04T08:15:12,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 87fdd579d0b26040c8abef46e9d1ae20, disabling compactions & flushes 2023-10-04T08:15:12,326 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:15:12,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:15:12,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. after waiting 0 ms 2023-10-04T08:15:12,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:15:12,326 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 87fdd579d0b26040c8abef46e9d1ae20 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:12,338 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/.tmp/testFamily/3dafecd214504673a235f28fd3ab58e4 is 61, key is row-0/testFamily:testQualifier/1696407163740/Put/seqid=0 2023-10-04T08:15:12,384 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:12,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:15:12,584 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 87fdd579d0b26040c8abef46e9d1ae20, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:12,745 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/.tmp/testFamily/3dafecd214504673a235f28fd3ab58e4 2023-10-04T08:15:12,751 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/.tmp/testFamily/3dafecd214504673a235f28fd3ab58e4 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/testFamily/3dafecd214504673a235f28fd3ab58e4 2023-10-04T08:15:12,757 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/testFamily/3dafecd214504673a235f28fd3ab58e4, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:12,758 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 87fdd579d0b26040c8abef46e9d1ae20 in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:12,763 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNext_0_-false/87fdd579d0b26040c8abef46e9d1ae20/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:12,764 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 87fdd579d0b26040c8abef46e9d1ae20: 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1696407162718.87fdd579d0b26040c8abef46e9d1ae20. 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9c208f35f121b8b6fe14f4139d19c72e, disabling compactions & flushes 2023-10-04T08:15:12,764 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. after waiting 0 ms 2023-10-04T08:15:12,764 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:15:12,764 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9c208f35f121b8b6fe14f4139d19c72e 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:12,777 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/.tmp/testFamily/d0617cf8ac5a4d1784a23280cda52a86 is 61, key is row-0/testFamily:testQualifier/1696407107901/Put/seqid=0 2023-10-04T08:15:12,784 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:12,985 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:13,044 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2023-10-04T08:15:13,184 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/.tmp/testFamily/d0617cf8ac5a4d1784a23280cda52a86 2023-10-04T08:15:13,185 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 13 regions to close 2023-10-04T08:15:13,185 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, 9c208f35f121b8b6fe14f4139d19c72e, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:13,191 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/.tmp/testFamily/d0617cf8ac5a4d1784a23280cda52a86 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/testFamily/d0617cf8ac5a4d1784a23280cda52a86 2023-10-04T08:15:13,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/testFamily/d0617cf8ac5a4d1784a23280cda52a86, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:13,197 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9c208f35f121b8b6fe14f4139d19c72e in 433ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:13,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/9c208f35f121b8b6fe14f4139d19c72e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:13,202 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:15:13,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9c208f35f121b8b6fe14f4139d19c72e: 2023-10-04T08:15:13,202 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1696407104845.9c208f35f121b8b6fe14f4139d19c72e. 2023-10-04T08:15:13,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 59fbb4b736d248cd85fe604c5ee58655, disabling compactions & flushes 2023-10-04T08:15:13,203 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:13,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:13,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. after waiting 0 ms 2023-10-04T08:15:13,203 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:13,203 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 59fbb4b736d248cd85fe604c5ee58655 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:13,215 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/.tmp/testFamily/b00e3e7754ef4e58a0eea33cf4aa9f6e is 61, key is row-0/testFamily:testQualifier/1696407244632/Put/seqid=0 2023-10-04T08:15:13,385 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:13,586 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 59fbb4b736d248cd85fe604c5ee58655, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:13,621 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/.tmp/testFamily/b00e3e7754ef4e58a0eea33cf4aa9f6e 2023-10-04T08:15:13,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/.tmp/testFamily/b00e3e7754ef4e58a0eea33cf4aa9f6e as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/testFamily/b00e3e7754ef4e58a0eea33cf4aa9f6e 2023-10-04T08:15:13,633 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/testFamily/b00e3e7754ef4e58a0eea33cf4aa9f6e, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:13,634 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 59fbb4b736d248cd85fe604c5ee58655 in 431ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:13,640 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnNextAsync_1_-true/59fbb4b736d248cd85fe604c5ee58655/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:13,641 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:13,641 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 59fbb4b736d248cd85fe604c5ee58655: 2023-10-04T08:15:13,641 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1696407241615.59fbb4b736d248cd85fe604c5ee58655. 2023-10-04T08:15:13,642 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0eeaa61c399a56345cddfcafd2aa53ca, disabling compactions & flushes 2023-10-04T08:15:13,642 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:13,642 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:13,642 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. after waiting 0 ms 2023-10-04T08:15:13,642 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:13,642 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 0eeaa61c399a56345cddfcafd2aa53ca 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:13,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/.tmp/testFamily/6da477f51c1947e08e888107b4c69fba is 61, key is row-0/testFamily:testQualifier/1696407303131/Put/seqid=0 2023-10-04T08:15:13,786 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:13,986 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 0eeaa61c399a56345cddfcafd2aa53ca, 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:14,060 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/.tmp/testFamily/6da477f51c1947e08e888107b4c69fba 2023-10-04T08:15:14,067 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/.tmp/testFamily/6da477f51c1947e08e888107b4c69fba as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/testFamily/6da477f51c1947e08e888107b4c69fba 2023-10-04T08:15:14,072 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/testFamily/6da477f51c1947e08e888107b4c69fba, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:14,074 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0eeaa61c399a56345cddfcafd2aa53ca in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:14,079 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/0eeaa61c399a56345cddfcafd2aa53ca/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:14,080 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:14,080 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0eeaa61c399a56345cddfcafd2aa53ca: 2023-10-04T08:15:14,080 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1696407300106.0eeaa61c399a56345cddfcafd2aa53ca. 2023-10-04T08:15:14,080 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 6438062196d04176b2d1947e41c0d3e6, disabling compactions & flushes 2023-10-04T08:15:14,081 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:14,081 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:14,081 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. after waiting 0 ms 2023-10-04T08:15:14,081 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:14,081 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 6438062196d04176b2d1947e41c0d3e6 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:14,093 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/.tmp/testFamily/12a428a8037146809706b55da69f307e is 61, key is row-0/testFamily:testQualifier/1696407233475/Put/seqid=0 2023-10-04T08:15:14,186 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 10 regions to close 2023-10-04T08:15:14,186 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:14,387 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 6438062196d04176b2d1947e41c0d3e6, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:14,498 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/.tmp/testFamily/12a428a8037146809706b55da69f307e 2023-10-04T08:15:14,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/.tmp/testFamily/12a428a8037146809706b55da69f307e as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/testFamily/12a428a8037146809706b55da69f307e 2023-10-04T08:15:14,510 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/testFamily/12a428a8037146809706b55da69f307e, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:14,511 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6438062196d04176b2d1947e41c0d3e6 in 430ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:14,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnRenewLease_1_-true/6438062196d04176b2d1947e41c0d3e6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:14,516 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:14,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 6438062196d04176b2d1947e41c0d3e6: 2023-10-04T08:15:14,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1696407230457.6438062196d04176b2d1947e41c0d3e6. 2023-10-04T08:15:14,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8b44dd21679fb8b47ec68172b7a9d6ab, disabling compactions & flushes 2023-10-04T08:15:14,516 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:15:14,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:15:14,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. after waiting 0 ms 2023-10-04T08:15:14,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:15:14,517 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8b44dd21679fb8b47ec68172b7a9d6ab 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:14,528 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/.tmp/testFamily/84d1f05390d04f2ca557132e614754c1 is 61, key is row-0/testFamily:testQualifier/1696407121149/Put/seqid=0 2023-10-04T08:15:14,587 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:14,787 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 8b44dd21679fb8b47ec68172b7a9d6ab, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:14,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/.tmp/testFamily/84d1f05390d04f2ca557132e614754c1 2023-10-04T08:15:14,940 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/.tmp/testFamily/84d1f05390d04f2ca557132e614754c1 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/testFamily/84d1f05390d04f2ca557132e614754c1 2023-10-04T08:15:14,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/testFamily/84d1f05390d04f2ca557132e614754c1, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:14,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b44dd21679fb8b47ec68172b7a9d6ab in 429ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/8b44dd21679fb8b47ec68172b7a9d6ab/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:14,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8b44dd21679fb8b47ec68172b7a9d6ab: 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1696407120127.8b44dd21679fb8b47ec68172b7a9d6ab. 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a86ff5fbeda7bbb8ff18cf9e11cfdf8e, disabling compactions & flushes 2023-10-04T08:15:14,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. after waiting 0 ms 2023-10-04T08:15:14,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:15:14,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing a86ff5fbeda7bbb8ff18cf9e11cfdf8e 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:14,963 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/.tmp/testFamily/b00a915f251f404a939bfe65ee326491 is 61, key is row-0/testFamily:testQualifier/1696407278989/Put/seqid=0 2023-10-04T08:15:14,987 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:15,187 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-04T08:15:15,188 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1583): Online Regions={a86ff5fbeda7bbb8ff18cf9e11cfdf8e=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e., d8089a176acb4ff8d652d8d65d4e554a=hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a., 96dd4b21100c44df0f1d2c389effacfe=testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe., 7cf4420553a66064d10b73e18ff7ce65=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65., 87c60d1bc9734410abc9413183bad22a=testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a., 2b8b1fed447fcf6ffb03157a004d4e73=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73., 3443a3e340fdb17279ac1f79e8b74717=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., b535ccf633612976045f65cd48c88d1b=hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.} 2023-10-04T08:15:15,188 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:15,369 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/.tmp/testFamily/b00a915f251f404a939bfe65ee326491 2023-10-04T08:15:15,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/.tmp/testFamily/b00a915f251f404a939bfe65ee326491 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/testFamily/b00a915f251f404a939bfe65ee326491 2023-10-04T08:15:15,382 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/testFamily/b00a915f251f404a939bfe65ee326491, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:15,383 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a86ff5fbeda7bbb8ff18cf9e11cfdf8e in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:15,387 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/a86ff5fbeda7bbb8ff18cf9e11cfdf8e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:15,388 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 96dd4b21100c44df0f1d2c389effacfe, a86ff5fbeda7bbb8ff18cf9e11cfdf8e, b535ccf633612976045f65cd48c88d1b, d8089a176acb4ff8d652d8d65d4e554a 2023-10-04T08:15:15,388 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a86ff5fbeda7bbb8ff18cf9e11cfdf8e: 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1696407277972.a86ff5fbeda7bbb8ff18cf9e11cfdf8e. 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d8089a176acb4ff8d652d8d65d4e554a, disabling compactions & flushes 2023-10-04T08:15:15,388 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. after waiting 0 ms 2023-10-04T08:15:15,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:15:15,389 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing d8089a176acb4ff8d652d8d65d4e554a 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:15,401 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/.tmp/testFamily/b77458b3f6c544c4aeb1b66d739669ae is 61, key is row-0/testFamily:testQualifier/1696407151589/Put/seqid=0 2023-10-04T08:15:15,407 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/.tmp/testFamily/b77458b3f6c544c4aeb1b66d739669ae 2023-10-04T08:15:15,412 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/.tmp/testFamily/b77458b3f6c544c4aeb1b66d739669ae as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/testFamily/b77458b3f6c544c4aeb1b66d739669ae 2023-10-04T08:15:15,417 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/testFamily/b77458b3f6c544c4aeb1b66d739669ae, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:15,418 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d8089a176acb4ff8d652d8d65d4e554a in 29ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:15,422 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_0_-false/d8089a176acb4ff8d652d8d65d4e554a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:15,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d8089a176acb4ff8d652d8d65d4e554a: 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1696407150568.d8089a176acb4ff8d652d8d65d4e554a. 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 96dd4b21100c44df0f1d2c389effacfe, disabling compactions & flushes 2023-10-04T08:15:15,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. after waiting 0 ms 2023-10-04T08:15:15,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:15:15,423 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 96dd4b21100c44df0f1d2c389effacfe 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:15,435 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/.tmp/testFamily/2934c2884ce840fd8a7c7a4dfd7e8f30 is 61, key is row-0/testFamily:testQualifier/1696407141426/Put/seqid=0 2023-10-04T08:15:15,588 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 96dd4b21100c44df0f1d2c389effacfe, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:15,788 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, 96dd4b21100c44df0f1d2c389effacfe, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:15,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/.tmp/testFamily/2934c2884ce840fd8a7c7a4dfd7e8f30 2023-10-04T08:15:15,848 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/.tmp/testFamily/2934c2884ce840fd8a7c7a4dfd7e8f30 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/testFamily/2934c2884ce840fd8a7c7a4dfd7e8f30 2023-10-04T08:15:15,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/testFamily/2934c2884ce840fd8a7c7a4dfd7e8f30, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:15,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96dd4b21100c44df0f1d2c389effacfe in 431ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:15,859 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/96dd4b21100c44df0f1d2c389effacfe/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:15,859 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 96dd4b21100c44df0f1d2c389effacfe: 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1696407138404.96dd4b21100c44df0f1d2c389effacfe. 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7cf4420553a66064d10b73e18ff7ce65, disabling compactions & flushes 2023-10-04T08:15:15,860 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. after waiting 0 ms 2023-10-04T08:15:15,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:15:15,860 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7cf4420553a66064d10b73e18ff7ce65 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:15,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/.tmp/testFamily/60d8196c61f14d89a82ac73ee2a9861d is 61, key is row-0/testFamily:testQualifier/1696407218325/Put/seqid=0 2023-10-04T08:15:15,989 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:16,189 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 5 regions to close 2023-10-04T08:15:16,189 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1583): Online Regions={7cf4420553a66064d10b73e18ff7ce65=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65., 87c60d1bc9734410abc9413183bad22a=testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a., 2b8b1fed447fcf6ffb03157a004d4e73=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73., 3443a3e340fdb17279ac1f79e8b74717=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., b535ccf633612976045f65cd48c88d1b=hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.} 2023-10-04T08:15:16,189 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 7cf4420553a66064d10b73e18ff7ce65, 87c60d1bc9734410abc9413183bad22a, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:16,279 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/.tmp/testFamily/60d8196c61f14d89a82ac73ee2a9861d 2023-10-04T08:15:16,285 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/.tmp/testFamily/60d8196c61f14d89a82ac73ee2a9861d as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/testFamily/60d8196c61f14d89a82ac73ee2a9861d 2023-10-04T08:15:16,291 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/testFamily/60d8196c61f14d89a82ac73ee2a9861d, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:16,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7cf4420553a66064d10b73e18ff7ce65 in 432ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:16,296 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/7cf4420553a66064d10b73e18ff7ce65/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:16,297 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7cf4420553a66064d10b73e18ff7ce65: 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1696407217307.7cf4420553a66064d10b73e18ff7ce65. 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 87c60d1bc9734410abc9413183bad22a, disabling compactions & flushes 2023-10-04T08:15:16,297 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. after waiting 0 ms 2023-10-04T08:15:16,297 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:15:16,297 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 87c60d1bc9734410abc9413183bad22a 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:16,309 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/.tmp/testFamily/e68ce48e744b4f299804da6a5f07704d is 61, key is row-0/testFamily:testQualifier/1696407230395/Put/seqid=0 2023-10-04T08:15:16,389 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 87c60d1bc9734410abc9413183bad22a, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:16,589 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, 87c60d1bc9734410abc9413183bad22a, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:16,715 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/.tmp/testFamily/e68ce48e744b4f299804da6a5f07704d 2023-10-04T08:15:16,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/.tmp/testFamily/e68ce48e744b4f299804da6a5f07704d as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/testFamily/e68ce48e744b4f299804da6a5f07704d 2023-10-04T08:15:16,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/testFamily/e68ce48e744b4f299804da6a5f07704d, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:16,727 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 87c60d1bc9734410abc9413183bad22a in 430ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/default/testRetryOutOfOrderScannerNextException_1_-true/87c60d1bc9734410abc9413183bad22a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:16,732 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 87c60d1bc9734410abc9413183bad22a: 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1696407227376.87c60d1bc9734410abc9413183bad22a. 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2b8b1fed447fcf6ffb03157a004d4e73, disabling compactions & flushes 2023-10-04T08:15:16,732 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. after waiting 0 ms 2023-10-04T08:15:16,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:15:16,732 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2b8b1fed447fcf6ffb03157a004d4e73 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:16,744 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/.tmp/testFamily/4c1c1f25af2b4ac980d4a84da78a2a22 is 61, key is row-0/testFamily:testQualifier/1696407167847/Put/seqid=0 2023-10-04T08:15:16,790 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:16,990 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 2b8b1fed447fcf6ffb03157a004d4e73, 3443a3e340fdb17279ac1f79e8b74717, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:17,151 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/.tmp/testFamily/4c1c1f25af2b4ac980d4a84da78a2a22 2023-10-04T08:15:17,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/.tmp/testFamily/4c1c1f25af2b4ac980d4a84da78a2a22 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/testFamily/4c1c1f25af2b4ac980d4a84da78a2a22 2023-10-04T08:15:17,163 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/testFamily/4c1c1f25af2b4ac980d4a84da78a2a22, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:17,163 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2b8b1fed447fcf6ffb03157a004d4e73 in 431ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:17,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2b8b1fed447fcf6ffb03157a004d4e73/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:17,168 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:15:17,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2b8b1fed447fcf6ffb03157a004d4e73: 2023-10-04T08:15:17,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1696407164823.2b8b1fed447fcf6ffb03157a004d4e73. 2023-10-04T08:15:17,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3443a3e340fdb17279ac1f79e8b74717, disabling compactions & flushes 2023-10-04T08:15:17,169 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:15:17,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:15:17,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. after waiting 0 ms 2023-10-04T08:15:17,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:15:17,169 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3443a3e340fdb17279ac1f79e8b74717 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:17,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/.tmp/testFamily/ea43c26034d04553803d9866e61c462e is 61, key is row-0/testFamily:testQualifier/1696407208254/Put/seqid=0 2023-10-04T08:15:17,190 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-10-04T08:15:17,190 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1583): Online Regions={3443a3e340fdb17279ac1f79e8b74717=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717., b535ccf633612976045f65cd48c88d1b=hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b.} 2023-10-04T08:15:17,190 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 3443a3e340fdb17279ac1f79e8b74717, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:17,385 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2023-10-04T08:15:17,390 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 3443a3e340fdb17279ac1f79e8b74717, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:17,590 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/.tmp/testFamily/ea43c26034d04553803d9866e61c462e 2023-10-04T08:15:17,591 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on 3443a3e340fdb17279ac1f79e8b74717, b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:17,597 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/.tmp/testFamily/ea43c26034d04553803d9866e61c462e as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/testFamily/ea43c26034d04553803d9866e61c462e 2023-10-04T08:15:17,602 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/testFamily/ea43c26034d04553803d9866e61c462e, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:17,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3443a3e340fdb17279ac1f79e8b74717 in 434ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:17,607 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/3443a3e340fdb17279ac1f79e8b74717/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:17,608 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:15:17,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3443a3e340fdb17279ac1f79e8b74717: 2023-10-04T08:15:17,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1696407205236.3443a3e340fdb17279ac1f79e8b74717. 2023-10-04T08:15:17,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b535ccf633612976045f65cd48c88d1b, disabling compactions & flushes 2023-10-04T08:15:17,608 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:15:17,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:15:17,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. after waiting 0 ms 2023-10-04T08:15:17,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:15:17,609 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b535ccf633612976045f65cd48c88d1b 1/1 column families, dataSize=228 B heapSize=800 B 2023-10-04T08:15:17,621 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/.tmp/testFamily/deaad1aa868e42a5a280adb5e6cb18e3 is 61, key is row-0/testFamily:testQualifier/1696407250712/Put/seqid=0 2023-10-04T08:15:17,791 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:17,991 DEBUG [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1609): Waiting on b535ccf633612976045f65cd48c88d1b 2023-10-04T08:15:18,025 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/.tmp/testFamily/deaad1aa868e42a5a280adb5e6cb18e3 2023-10-04T08:15:18,032 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/.tmp/testFamily/deaad1aa868e42a5a280adb5e6cb18e3 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/testFamily/deaad1aa868e42a5a280adb5e6cb18e3 2023-10-04T08:15:18,038 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/testFamily/deaad1aa868e42a5a280adb5e6cb18e3, entries=4, sequenceid=8, filesize=4.9 K 2023-10-04T08:15:18,039 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b535ccf633612976045f65cd48c88d1b in 430ms, sequenceid=8, compaction requested=false 2023-10-04T08:15:18,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/data/hbase/testMetaScanTimeoutOnClose_1_-true/b535ccf633612976045f65cd48c88d1b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2023-10-04T08:15:18,045 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:15:18,045 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b535ccf633612976045f65cd48c88d1b: 2023-10-04T08:15:18,045 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1696407247690.b535ccf633612976045f65cd48c88d1b. 2023-10-04T08:15:18,191 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,45997,1696407041329; all regions closed. 2023-10-04T08:15:18,201 DEBUG [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/oldWALs 2023-10-04T08:15:18,201 INFO [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45997%2C1696407041329.meta:.meta(num 1696407103356) 2023-10-04T08:15:18,206 DEBUG [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/oldWALs 2023-10-04T08:15:18,206 INFO [RS:0;jenkins-hbase3:45997 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C45997%2C1696407041329:(num 1696407103016) 2023-10-04T08:15:18,206 DEBUG [RS:0;jenkins-hbase3:45997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:18,206 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.LeaseManager(133): Closed leases 2023-10-04T08:15:18,206 INFO [RS:0;jenkins-hbase3:45997 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-10-04T08:15:18,206 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-04T08:15:18,207 INFO [RS:0;jenkins-hbase3:45997 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:45997 2023-10-04T08:15:18,213 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase3.apache.org,45997,1696407041329 2023-10-04T08:15:18,213 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2023-10-04T08:15:18,215 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,45997,1696407041329] 2023-10-04T08:15:18,216 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,45997,1696407041329; numProcessing=1 2023-10-04T08:15:18,217 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase3.apache.org,45997,1696407041329 already deleted, retry=false 2023-10-04T08:15:18,217 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,45997,1696407041329 expired; onlineServers=0 2023-10-04T08:15:18,217 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,46181,1696407040117' ***** 2023-10-04T08:15:18,217 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-04T08:15:18,217 DEBUG [M:0;jenkins-hbase3:46181 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2092e586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-04T08:15:18,217 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,46181,1696407040117 2023-10-04T08:15:18,217 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,46181,1696407040117; all regions closed. 2023-10-04T08:15:18,217 DEBUG [M:0;jenkins-hbase3:46181 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-04T08:15:18,218 DEBUG [M:0;jenkins-hbase3:46181 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-04T08:15:18,218 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-04T08:15:18,218 DEBUG [M:0;jenkins-hbase3:46181 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-04T08:15:18,218 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696407042866 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696407042866,5,FailOnTimeoutGroup] 2023-10-04T08:15:18,218 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696407042862 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696407042862,5,FailOnTimeoutGroup] 2023-10-04T08:15:18,218 INFO [M:0;jenkins-hbase3:46181 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-04T08:15:18,218 DEBUG [M:0;jenkins-hbase3:46181 {}] master.HMaster(1688): Stopping service threads 2023-10-04T08:15:18,218 INFO [M:0;jenkins-hbase3:46181 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-04T08:15:18,219 ERROR [M:0;jenkins-hbase3:46181 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1301226619) connection to localhost/127.0.0.1:45837 from jenkins,5,PEWorkerGroup] 2023-10-04T08:15:18,219 INFO [M:0;jenkins-hbase3:46181 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-04T08:15:18,219 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-04T08:15:18,220 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2023-10-04T08:15:18,220 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2023-10-04T08:15:18,220 DEBUG [M:0;jenkins-hbase3:46181 {}] zookeeper.ZKUtil(347): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2023-10-04T08:15:18,221 WARN [M:0;jenkins-hbase3:46181 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-04T08:15:18,221 INFO [M:0;jenkins-hbase3:46181 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-04T08:15:18,221 INFO [M:0;jenkins-hbase3:46181 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-04T08:15:18,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2023-10-04T08:15:18,221 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-04T08:15:18,221 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:15:18,221 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:15:18,221 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-04T08:15:18,221 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:15:18,221 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=451.45 KB heapSize=522.68 KB 2023-10-04T08:15:18,234 DEBUG [M:0;jenkins-hbase3:46181 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32834ec7c3fc492c92734dd9b19d9640 is 88, key is hbase:meta,,1/info:sn/1696407103124/Put/seqid=0 2023-10-04T08:15:18,285 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-04T08:15:18,315 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-04T08:15:18,315 INFO [RS:0;jenkins-hbase3:45997 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,45997,1696407041329; zookeeper connection closed. 2023-10-04T08:15:18,315 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): regionserver:45997-0x1017c868d4b0001, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-04T08:15:18,315 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@759c7324 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@759c7324 2023-10-04T08:15:18,316 INFO [Listener at localhost/43533 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-04T08:15:18,639 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32834ec7c3fc492c92734dd9b19d9640 2023-10-04T08:15:18,660 DEBUG [M:0;jenkins-hbase3:46181 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8419ac8750e341bba3db41452f6a6342 is 827, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1696407106068/Put/seqid=0 2023-10-04T08:15:19,066 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=450.86 KB at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8419ac8750e341bba3db41452f6a6342 2023-10-04T08:15:19,072 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8419ac8750e341bba3db41452f6a6342 2023-10-04T08:15:19,085 DEBUG [M:0;jenkins-hbase3:46181 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a1aab2e91a87489cb3f5b78d0b825371 is 82, key is jenkins-hbase3.apache.org,45997,1696407041329/rs:state/1696407072877/Put/seqid=0 2023-10-04T08:15:19,089 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=828 (bloomFilter=true), to=hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a1aab2e91a87489cb3f5b78d0b825371 2023-10-04T08:15:19,094 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/32834ec7c3fc492c92734dd9b19d9640 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/32834ec7c3fc492c92734dd9b19d9640 2023-10-04T08:15:19,099 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/32834ec7c3fc492c92734dd9b19d9640, entries=8, sequenceid=828, filesize=5.5 K 2023-10-04T08:15:19,100 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8419ac8750e341bba3db41452f6a6342 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8419ac8750e341bba3db41452f6a6342 2023-10-04T08:15:19,104 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8419ac8750e341bba3db41452f6a6342 2023-10-04T08:15:19,104 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8419ac8750e341bba3db41452f6a6342, entries=92, sequenceid=828, filesize=29.5 K 2023-10-04T08:15:19,105 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a1aab2e91a87489cb3f5b78d0b825371 as hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a1aab2e91a87489cb3f5b78d0b825371 2023-10-04T08:15:19,109 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:45837/user/jenkins/test-data/3fc490e0-2575-f5f8-d1bd-3e85a9aedc62/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a1aab2e91a87489cb3f5b78d0b825371, entries=1, sequenceid=828, filesize=5.1 K 2023-10-04T08:15:19,110 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(3022): Finished flush of dataSize ~451.45 KB/462285, heapSize ~522.38 KB/534920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 889ms, sequenceid=828, compaction requested=false 2023-10-04T08:15:19,113 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-04T08:15:19,113 DEBUG [M:0;jenkins-hbase3:46181 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-04T08:15:19,115 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-04T08:15:19,115 INFO [M:0;jenkins-hbase3:46181 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-04T08:15:19,116 INFO [M:0;jenkins-hbase3:46181 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:46181 2023-10-04T08:15:19,117 DEBUG [M:0;jenkins-hbase3:46181 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase3.apache.org,46181,1696407040117 already deleted, retry=false 2023-10-04T08:15:19,219 INFO [M:0;jenkins-hbase3:46181 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,46181,1696407040117; zookeeper connection closed. 2023-10-04T08:15:19,220 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-04T08:15:19,220 DEBUG [Listener at localhost/43533-EventThread {}] zookeeper.ZKWatcher(604): master:46181-0x1017c868d4b0000, quorum=127.0.0.1:50391, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-04T08:15:19,221 WARN [Listener at localhost/43533 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-04T08:15:19,225 INFO [Listener at localhost/43533 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-04T08:15:19,329 WARN [BP-1722339665-172.31.12.81-1696407037142 heartbeating to localhost/127.0.0.1:45837 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-04T08:15:19,329 WARN [BP-1722339665-172.31.12.81-1696407037142 heartbeating to localhost/127.0.0.1:45837 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1722339665-172.31.12.81-1696407037142 (Datanode Uuid 588ecdd4-8ff0-4951-ab0b-b49dcb70b795) service to localhost/127.0.0.1:45837 2023-10-04T08:15:19,331 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181/dfs/data/data1/current/BP-1722339665-172.31.12.81-1696407037142 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-04T08:15:19,331 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b4bb295f-6233-dd66-a95c-3b82a754fea0/cluster_e65888a6-1b33-e96c-06e7-20ba3555d181/dfs/data/data2/current/BP-1722339665-172.31.12.81-1696407037142 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-04T08:15:19,357 INFO [Listener at localhost/43533 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-04T08:15:19,473 INFO [Listener at localhost/43533 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-10-04T08:15:19,520 INFO [Listener at localhost/43533 {}] hbase.HBaseTestingUtility(1347): Minicluster is down