2024-01-01 09:08:47,815 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4988d8b8 2024-01-01 09:08:47,827 main DEBUG Took 0.010110 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-01-01 09:08:47,827 main DEBUG PluginManager 'Core' found 129 plugins 2024-01-01 09:08:47,827 main DEBUG PluginManager 'Level' found 0 plugins 2024-01-01 09:08:47,828 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-01-01 09:08:47,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,837 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-01-01 09:08:47,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,850 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,851 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,851 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,852 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,853 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,854 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,855 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,855 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,856 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,856 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,857 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,857 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,858 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-01-01 09:08:47,858 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,858 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-01-01 09:08:47,860 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-01-01 09:08:47,862 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-01-01 09:08:47,864 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-01-01 09:08:47,864 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-01-01 09:08:47,865 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-01-01 09:08:47,865 main DEBUG PluginManager 'Converter' found 47 plugins 2024-01-01 09:08:47,875 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-01-01 09:08:47,878 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-01-01 09:08:47,879 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-01-01 09:08:47,879 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-01-01 09:08:47,882 main DEBUG createAppenders(={Console}) 2024-01-01 09:08:47,883 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4988d8b8 initialized 2024-01-01 09:08:47,883 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4988d8b8 2024-01-01 09:08:47,883 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4988d8b8 OK. 2024-01-01 09:08:47,884 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-01-01 09:08:47,884 main DEBUG OutputStream closed 2024-01-01 09:08:47,884 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-01-01 09:08:47,885 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-01-01 09:08:47,885 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@436813f3 OK 2024-01-01 09:08:47,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-01-01 09:08:47,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-01-01 09:08:47,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-01-01 09:08:47,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-01-01 09:08:47,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-01-01 09:08:47,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-01-01 09:08:47,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-01-01 09:08:47,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-01-01 09:08:47,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-01-01 09:08:47,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-01-01 09:08:47,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-01-01 09:08:47,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-01-01 09:08:47,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-01-01 09:08:47,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-01-01 09:08:47,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-01-01 09:08:47,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-01-01 09:08:47,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-01-01 09:08:47,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-01-01 09:08:47,938 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-01-01 09:08:47,939 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7c711375) with optional ClassLoader: null 2024-01-01 09:08:47,939 main DEBUG Shutdown hook enabled. Registering a new one. 2024-01-01 09:08:47,940 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@7c711375] started OK. 2024-01-01T09:08:48,098 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-01-01 09:08:48,102 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-01-01 09:08:48,102 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-01-01T09:08:48,191 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24 2024-01-01T09:08:48,229 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-01-01T09:08:48,233 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864, deleteOnExit=true 2024-01-01T09:08:48,233 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-01-01T09:08:48,234 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/test.cache.data in system properties and HBase conf 2024-01-01T09:08:48,234 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/hadoop.tmp.dir in system properties and HBase conf 2024-01-01T09:08:48,235 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/hadoop.log.dir in system properties and HBase conf 2024-01-01T09:08:48,235 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/mapreduce.cluster.local.dir in system properties and HBase conf 2024-01-01T09:08:48,236 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-01-01T09:08:48,236 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-01-01T09:08:48,341 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-01-01T09:08:48,746 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-01-01T09:08:48,750 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-01-01T09:08:48,750 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-01-01T09:08:48,750 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-01-01T09:08:48,751 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-01-01T09:08:48,751 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-01-01T09:08:48,751 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-01-01T09:08:48,751 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-01-01T09:08:48,752 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/dfs.journalnode.edits.dir in system properties and HBase conf 2024-01-01T09:08:48,752 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-01-01T09:08:48,752 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/nfs.dump.dir in system properties and HBase conf 2024-01-01T09:08:48,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/java.io.tmpdir in system properties and HBase conf 2024-01-01T09:08:48,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/dfs.journalnode.edits.dir in system properties and HBase conf 2024-01-01T09:08:48,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-01-01T09:08:48,753 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2024-01-01T09:08:49,215 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-01-01T09:08:49,219 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-01-01T09:08:49,475 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-01-01T09:08:49,600 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3ae437a9 via org.mortbay.log.Slf4jLog 2024-01-01T09:08:49,613 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-01-01T09:08:49,648 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2024-01-01T09:08:49,680 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/java.io.tmpdir/Jetty_localhost_34355_hdfs____g69cxo/webapp 2024-01-01T09:08:49,806 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34355 2024-01-01T09:08:49,826 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-01-01T09:08:49,826 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-01-01T09:08:50,368 WARN [Listener at localhost/43719 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-01-01T09:08:50,427 WARN [Listener at localhost/43719 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-01-01T09:08:50,446 WARN [Listener at localhost/43719 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-01-01T09:08:50,452 INFO [Listener at localhost/43719 {}] log.Slf4jLog(67): jetty-6.1.26 2024-01-01T09:08:50,457 INFO [Listener at localhost/43719 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/java.io.tmpdir/Jetty_localhost_44241_datanode____.92d4ve/webapp 2024-01-01T09:08:50,558 INFO [Listener at localhost/43719 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44241 2024-01-01T09:08:50,763 WARN [Listener at localhost/38993 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-01-01T09:08:51,034 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-01-01T09:08:51,230 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa43e859d094d0e56: Processing first storage report for DS-7db2438d-1b04-421f-9cdd-77776f01c743 from datanode 0abf6536-c559-4747-8986-c6b584d20a6d 2024-01-01T09:08:51,231 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa43e859d094d0e56: from storage DS-7db2438d-1b04-421f-9cdd-77776f01c743 node DatanodeRegistration(127.0.0.1:34177, datanodeUuid=0abf6536-c559-4747-8986-c6b584d20a6d, infoPort=46779, infoSecurePort=0, ipcPort=38993, storageInfo=lv=-57;cid=testClusterID;nsid=1285442933;c=1704100129287), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-01-01T09:08:51,232 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa43e859d094d0e56: Processing first storage report for DS-550cf48a-b765-4bc4-aa19-7ca5ddfae6c8 from datanode 0abf6536-c559-4747-8986-c6b584d20a6d 2024-01-01T09:08:51,232 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa43e859d094d0e56: from storage DS-550cf48a-b765-4bc4-aa19-7ca5ddfae6c8 node DatanodeRegistration(127.0.0.1:34177, datanodeUuid=0abf6536-c559-4747-8986-c6b584d20a6d, infoPort=46779, infoSecurePort=0, ipcPort=38993, storageInfo=lv=-57;cid=testClusterID;nsid=1285442933;c=1704100129287), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-01-01T09:08:51,243 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24 2024-01-01T09:08:51,370 INFO [Listener at localhost/38993 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864/zookeeper_0, clientPort=56255, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-01-01T09:08:51,389 INFO [Listener at localhost/38993 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56255 2024-01-01T09:08:51,396 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:51,398 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:52,066 INFO [Listener at localhost/38993 {}] util.FSUtils(462): Created version file at hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 with version=8 2024-01-01T09:08:52,066 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/hbase-staging 2024-01-01T09:08:52,260 DEBUG [Listener at localhost/38993 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-01-01T09:08:52,876 INFO [Listener at localhost/38993 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=9 2024-01-01T09:08:52,909 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:52,909 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:52,909 INFO [Listener at localhost/38993 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-01-01T09:08:52,909 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:52,910 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-01-01T09:08:53,055 INFO [Listener at localhost/38993 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2024-01-01T09:08:53,155 INFO [Listener at localhost/38993 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-01-01T09:08:53,171 DEBUG [Listener at localhost/38993 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-01-01T09:08:53,177 INFO [Listener at localhost/38993 {}] ipc.NettyRpcServer(312): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-01-01T09:08:53,223 DEBUG [Listener at localhost/38993 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 6593 (auto-detected) 2024-01-01T09:08:53,224 DEBUG [Listener at localhost/38993 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2024-01-01T09:08:53,261 INFO [Listener at localhost/38993 {}] ipc.NettyRpcServer(194): Bind to /172.31.12.81:46807 2024-01-01T09:08:53,272 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:53,274 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:53,292 INFO [Listener at localhost/38993 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:46807 connecting to ZooKeeper ensemble=127.0.0.1:56255 2024-01-01T09:08:53,334 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:468070x0, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-01-01T09:08:53,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:46807-0x1014e62600b0000 connected 2024-01-01T09:08:53,367 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-01-01T09:08:53,370 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-01-01T09:08:53,374 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-01-01T09:08:53,381 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46807 2024-01-01T09:08:53,381 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46807 2024-01-01T09:08:53,382 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46807 2024-01-01T09:08:53,383 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46807 2024-01-01T09:08:53,383 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46807 2024-01-01T09:08:53,394 INFO [Listener at localhost/38993 {}] master.HMaster(487): hbase.rootdir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86, hbase.cluster.distributed=false 2024-01-01T09:08:53,452 INFO [Listener at localhost/38993 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=9 2024-01-01T09:08:53,452 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:53,452 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:53,453 INFO [Listener at localhost/38993 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-01-01T09:08:53,453 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-01-01T09:08:53,453 INFO [Listener at localhost/38993 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-01-01T09:08:53,458 INFO [Listener at localhost/38993 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2024-01-01T09:08:53,460 INFO [Listener at localhost/38993 {}] ipc.NettyRpcServer(312): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-01-01T09:08:53,461 INFO [Listener at localhost/38993 {}] ipc.NettyRpcServer(194): Bind to /172.31.12.81:44183 2024-01-01T09:08:53,463 INFO [Listener at localhost/38993 {}] hfile.BlockCacheFactory(116): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2024-01-01T09:08:53,469 DEBUG [Listener at localhost/38993 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-01-01T09:08:53,470 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:53,473 INFO [Listener at localhost/38993 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:53,476 INFO [Listener at localhost/38993 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44183 connecting to ZooKeeper ensemble=127.0.0.1:56255 2024-01-01T09:08:53,480 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:441830x0, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-01-01T09:08:53,481 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44183-0x1014e62600b0001 connected 2024-01-01T09:08:53,482 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-01-01T09:08:53,483 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-01-01T09:08:53,484 DEBUG [Listener at localhost/38993 {}] zookeeper.ZKUtil(113): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-01-01T09:08:53,485 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44183 2024-01-01T09:08:53,485 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44183 2024-01-01T09:08:53,485 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44183 2024-01-01T09:08:53,486 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44183 2024-01-01T09:08:53,486 DEBUG [Listener at localhost/38993 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44183 2024-01-01T09:08:53,488 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2395): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:53,497 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-01-01T09:08:53,497 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-01-01T09:08:53,500 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:53,510 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:46807 2024-01-01T09:08:53,518 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-01-01T09:08:53,518 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-01-01T09:08:53,518 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:53,518 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:53,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-01-01T09:08:53,521 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase3.apache.org,46807,1704100132244 from backup master directory 2024-01-01T09:08:53,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-01-01T09:08:53,524 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:53,524 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-01-01T09:08:53,524 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-01-01T09:08:53,525 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-01-01T09:08:53,525 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:53,527 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2024-01-01T09:08:53,530 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2024-01-01T09:08:53,604 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/hbase.id with ID: 3e51ff93-cf7c-4b2c-b033-f58fecb67ca0 2024-01-01T09:08:53,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-01-01T09:08:53,654 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:53,654 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:53,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:08:53,693 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-01-01T09:08:53,708 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2024-01-01T09:08:53,708 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2024-01-01T09:08:53,710 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:953) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2420) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:576) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:573) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-01-01T09:08:53,718 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:953) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2420) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:576) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:573) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-01-01T09:08:53,719 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-01-01T09:08:53,778 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store 2024-01-01T09:08:53,799 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-01-01T09:08:53,799 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:08:53,799 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-01-01T09:08:53,800 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:08:53,800 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:08:53,800 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-01-01T09:08:53,800 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:08:53,800 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:08:53,800 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-01-01T09:08:53,802 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/.initializing 2024-01-01T09:08:53,802 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/WALs/jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:53,810 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-01-01T09:08:53,822 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46807%2C1704100132244, suffix=, logDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/WALs/jenkins-hbase3.apache.org,46807,1704100132244, archiveDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/oldWALs, maxLogs=10 2024-01-01T09:08:53,848 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/WALs/jenkins-hbase3.apache.org,46807,1704100132244/jenkins-hbase3.apache.org%2C46807%2C1704100132244.1704100133829, exclude list is [], retry=0 2024-01-01T09:08:53,876 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK] 2024-01-01T09:08:53,883 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-01-01T09:08:53,951 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/WALs/jenkins-hbase3.apache.org,46807,1704100132244/jenkins-hbase3.apache.org%2C46807%2C1704100132244.1704100133829 2024-01-01T09:08:53,953 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK]] 2024-01-01T09:08:53,954 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:08:53,954 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:08:53,959 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:53,960 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-01-01T09:08:54,049 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:08:54,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-01-01T09:08:54,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:08:54,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-01-01T09:08:54,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:08:54,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-01-01T09:08:54,064 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:08:54,069 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,070 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,081 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-01-01T09:08:54,084 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-01-01T09:08:54,088 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:08:54,089 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65913863, jitterRate=-0.017806902527809143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-01-01T09:08:54,094 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-01-01T09:08:54,095 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-01-01T09:08:54,134 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12fa3e79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:08:54,181 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(862): No meta location available on zookeeper, skip migrating... 2024-01-01T09:08:54,197 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-01-01T09:08:54,197 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(610): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-01-01T09:08:54,200 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-01-01T09:08:54,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(630): Recovered RegionProcedureStore lease in 1 msec 2024-01-01T09:08:54,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(644): Loaded RegionProcedureStore in 5 msec 2024-01-01T09:08:54,208 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-01-01T09:08:54,240 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-01-01T09:08:54,255 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-01-01T09:08:54,257 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2024-01-01T09:08:54,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-01-01T09:08:54,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-01-01T09:08:54,264 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2024-01-01T09:08:54,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-01-01T09:08:54,271 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-01-01T09:08:54,272 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2024-01-01T09:08:54,273 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-01-01T09:08:54,275 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2024-01-01T09:08:54,287 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-01-01T09:08:54,289 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-01-01T09:08:54,294 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-01-01T09:08:54,294 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-01-01T09:08:54,294 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,295 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(806): Active/primary master=jenkins-hbase3.apache.org,46807,1704100132244, sessionid=0x1014e62600b0000, setting cluster-up flag (Was=false) 2024-01-01T09:08:54,310 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,310 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,318 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-01-01T09:08:54,320 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:54,325 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,326 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:08:54,333 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-01-01T09:08:54,334 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:08:54,399 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:44183 2024-01-01T09:08:54,401 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1004): ClusterId : 3e51ff93-cf7c-4b2c-b033-f58fecb67ca0 2024-01-01T09:08:54,404 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-01-01T09:08:54,410 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-01-01T09:08:54,410 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-01-01T09:08:54,418 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-01-01T09:08:54,419 DEBUG [RS:0;jenkins-hbase3:44183 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509d9c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:08:54,421 DEBUG [RS:0;jenkins-hbase3:44183 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34833867, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2024-01-01T09:08:54,426 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-01-01T09:08:54,427 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-01-01T09:08:54,427 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1086): About to register with Master. 2024-01-01T09:08:54,430 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3064): reportForDuty to master=jenkins-hbase3.apache.org,46807,1704100132244 with isa=jenkins-hbase3.apache.org/172.31.12.81:44183, startcode=1704100133450 2024-01-01T09:08:54,449 DEBUG [RS:0;jenkins-hbase3:44183 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-01-01T09:08:54,451 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1079): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-01-01T09:08:54,459 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-01-01T09:08:54,465 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-01-01T09:08:54,473 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,46807,1704100132244 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-01-01T09:08:54,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2024-01-01T09:08:54,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2024-01-01T09:08:54,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2024-01-01T09:08:54,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2024-01-01T09:08:54,480 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2024-01-01T09:08:54,481 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:08:54,481 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2024-01-01T09:08:54,481 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:08:54,483 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1704100164483 2024-01-01T09:08:54,486 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-01-01T09:08:54,495 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-01-01T09:08:54,497 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-01-01T09:08:54,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-01-01T09:08:54,505 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,505 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-01-01T09:08:54,509 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-01-01T09:08:54,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-01-01T09:08:54,511 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-01-01T09:08:54,511 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-01-01T09:08:54,512 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-01-01T09:08:54,516 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55647, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-01-01T09:08:54,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-01-01T09:08:54,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-01-01T09:08:54,520 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-01-01T09:08:54,526 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-01-01T09:08:54,527 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-01-01T09:08:54,532 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3225) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17345) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-01-01T09:08:54,532 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-01-01T09:08:54,532 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:08:54,537 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1704100134537,5,FailOnTimeoutGroup] 2024-01-01T09:08:54,538 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1704100134537,5,FailOnTimeoutGroup] 2024-01-01T09:08:54,538 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-01-01T09:08:54,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1641): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-01-01T09:08:54,544 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-01-01T09:08:54,544 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-01-01T09:08:54,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:08:54,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-01-01T09:08:54,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-01-01T09:08:54,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:08:54,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-01-01T09:08:54,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-01-01T09:08:54,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:08:54,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-01-01T09:08:54,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-01-01T09:08:54,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:08:54,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:08:54,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740 2024-01-01T09:08:54,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740 2024-01-01T09:08:54,568 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3083): Master is not running yet 2024-01-01T09:08:54,568 WARN [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1095): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-01-01T09:08:54,569 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-01-01T09:08:54,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2024-01-01T09:08:54,574 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:08:54,575 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73512642, jitterRate=0.09542372822761536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-01-01T09:08:54,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2024-01-01T09:08:54,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2024-01-01T09:08:54,577 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2024-01-01T09:08:54,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-01-01T09:08:54,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-01-01T09:08:54,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2024-01-01T09:08:54,579 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2024-01-01T09:08:54,579 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2024-01-01T09:08:54,583 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-01-01T09:08:54,583 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-01-01T09:08:54,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-01-01T09:08:54,607 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-01-01T09:08:54,610 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-01-01T09:08:54,761 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:08:55,762 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:08:56,763 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:08:57,763 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:08:58,764 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:08:59,764 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:00,765 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:01,765 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:02,766 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:03,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-01-01T09:09:03,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-01-01T09:09:03,766 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:04,767 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:05,767 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:06,768 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:07,768 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:08,769 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:09,770 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:10,770 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:11,771 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:12,771 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:13,772 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:14,772 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:15,773 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:16,773 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:17,774 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:18,774 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:19,775 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:20,775 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:21,776 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:22,776 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:23,777 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:24,424 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-01-01T09:09:24,429 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33842, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-01-01T09:09:24,569 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3064): reportForDuty to master=jenkins-hbase3.apache.org,46807,1704100132244 with isa=jenkins-hbase3.apache.org/172.31.12.81:44183, startcode=1704100133450 2024-01-01T09:09:24,575 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46807 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:24,585 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1721): Config from master: hbase.rootdir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:09:24,585 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1721): Config from master: fs.defaultFS=hdfs://localhost:43719 2024-01-01T09:09:24,585 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1721): Config from master: hbase.master.info.port=-1 2024-01-01T09:09:24,590 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-01-01T09:09:24,591 DEBUG [RS:0;jenkins-hbase3:44183 {}] zookeeper.ZKUtil(111): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:24,591 WARN [RS:0;jenkins-hbase3:44183 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-01-01T09:09:24,591 INFO [RS:0;jenkins-hbase3:44183 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-01-01T09:09:24,591 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(2097): logDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:24,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,44183,1704100133450] 2024-01-01T09:09:24,607 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-01-01T09:09:24,621 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-01-01T09:09:24,633 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2024-01-01T09:09:24,637 INFO [RS:0;jenkins-hbase3:44183 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-01-01T09:09:24,637 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,638 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer$CompactionChecker(1984): CompactionChecker runs every PT1S 2024-01-01T09:09:24,646 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2024-01-01T09:09:24,647 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2024-01-01T09:09:24,648 DEBUG [RS:0;jenkins-hbase3:44183 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2024-01-01T09:09:24,649 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,649 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,649 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,649 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,649 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44183,1704100133450-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-01-01T09:09:24,664 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-01-01T09:09:24,668 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44183,1704100133450-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:24,688 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,44183,1704100133450 started 2024-01-01T09:09:24,688 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1763): Serving as jenkins-hbase3.apache.org,44183,1704100133450, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:44183, sessionid=0x1014e62600b0001 2024-01-01T09:09:24,688 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-01-01T09:09:24,688 DEBUG [RS:0;jenkins-hbase3:44183 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:24,688 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44183,1704100133450' 2024-01-01T09:09:24,688 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-01-01T09:09:24,689 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-01-01T09:09:24,690 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-01-01T09:09:24,690 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-01-01T09:09:24,690 DEBUG [RS:0;jenkins-hbase3:44183 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:24,690 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44183,1704100133450' 2024-01-01T09:09:24,690 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-01-01T09:09:24,691 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-01-01T09:09:24,692 DEBUG [RS:0;jenkins-hbase3:44183 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-01-01T09:09:24,692 INFO [RS:0;jenkins-hbase3:44183 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2024-01-01T09:09:24,692 INFO [RS:0;jenkins-hbase3:44183 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-01-01T09:09:24,778 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:25,778 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:26,779 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:27,779 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:28,780 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:29,780 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:30,781 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:31,781 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:32,782 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:33,782 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:34,783 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:35,784 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:36,784 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:37,785 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:38,785 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:39,786 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:40,786 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:41,787 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:42,787 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:43,788 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:44,788 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:45,789 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:46,790 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:47,790 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:48,791 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:49,791 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:50,792 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:51,792 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:52,793 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:53,793 WARN [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-01-01T09:09:54,700 INFO [RS:0;jenkins-hbase3:44183 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-01-01T09:09:54,706 INFO [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44183%2C1704100133450, suffix=, logDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450, archiveDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/oldWALs, maxLogs=32 2024-01-01T09:09:54,719 DEBUG [RS:0;jenkins-hbase3:44183 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450/jenkins-hbase3.apache.org%2C44183%2C1704100133450.1704100194708, exclude list is [], retry=0 2024-01-01T09:09:54,725 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK] 2024-01-01T09:09:54,730 INFO [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450/jenkins-hbase3.apache.org%2C44183%2C1704100133450.1704100194708 2024-01-01T09:09:54,731 DEBUG [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK]] 2024-01-01T09:09:54,795 DEBUG [jenkins-hbase3:46807 {}] assignment.AssignmentManager(2361): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-01-01T09:09:54,802 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:54,807 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44183,1704100133450, state=OPENING 2024-01-01T09:09:54,814 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-01-01T09:09:54,815 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:09:54,815 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:09:54,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-01-01T09:09:54,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-01-01T09:09:54,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:09:55,003 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:55,005 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2024-01-01T09:09:55,008 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42780, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-01-01T09:09:55,022 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2024-01-01T09:09:55,022 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-01-01T09:09:55,023 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-01-01T09:09:55,025 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44183%2C1704100133450.meta, suffix=.meta, logDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450, archiveDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/oldWALs, maxLogs=32 2024-01-01T09:09:55,038 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450/jenkins-hbase3.apache.org%2C44183%2C1704100133450.meta.1704100195027.meta, exclude list is [], retry=0 2024-01-01T09:09:55,042 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK] 2024-01-01T09:09:55,045 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/WALs/jenkins-hbase3.apache.org,44183,1704100133450/jenkins-hbase3.apache.org%2C44183%2C1704100133450.meta.1704100195027.meta 2024-01-01T09:09:55,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34177,DS-7db2438d-1b04-421f-9cdd-77776f01c743,DISK]] 2024-01-01T09:09:55,046 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:09:55,047 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-01-01T09:09:55,065 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-01-01T09:09:55,072 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-01-01T09:09:55,078 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-01-01T09:09:55,078 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:55,078 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2024-01-01T09:09:55,078 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2024-01-01T09:09:55,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-01-01T09:09:55,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-01-01T09:09:55,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:55,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:09:55,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-01-01T09:09:55,084 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-01-01T09:09:55,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:55,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:09:55,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-01-01T09:09:55,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-01-01T09:09:55,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:55,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-01-01T09:09:55,089 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740 2024-01-01T09:09:55,091 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740 2024-01-01T09:09:55,093 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-01-01T09:09:55,095 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2024-01-01T09:09:55,097 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74899344, jitterRate=0.11608719825744629}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-01-01T09:09:55,098 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2024-01-01T09:09:55,108 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1704100194995 2024-01-01T09:09:55,124 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:meta,,1.1588230740 2024-01-01T09:09:55,125 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2024-01-01T09:09:55,126 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:55,129 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44183,1704100133450, state=OPEN 2024-01-01T09:09:55,139 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-01-01T09:09:55,139 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-01-01T09:09:55,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-01-01T09:09:55,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-01-01T09:09:55,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=3, resume processing ppid=2 2024-01-01T09:09:55,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44183,1704100133450 in 320 msec 2024-01-01T09:09:55,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=2, resume processing ppid=1 2024-01-01T09:09:55,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.552 sec 2024-01-01T09:09:55,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1 mins, 0.772 sec 2024-01-01T09:09:55,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1065): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1704100195157, completionTime=-1 2024-01-01T09:09:55,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(821): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-01-01T09:09:55,157 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joining cluster... 2024-01-01T09:09:55,213 DEBUG [hconnection-0x19f6bc01-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:09:55,215 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35432, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:09:55,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1676): Number of RegionServers=1 2024-01-01T09:09:55,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1704100255228 2024-01-01T09:09:55,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1704100315228 2024-01-01T09:09:55,228 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1683): Joined the cluster in 70 msec 2024-01-01T09:09:55,250 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:55,251 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:55,251 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:55,252 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:46807, period=300000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:55,253 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:55,259 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2024-01-01T09:09:55,267 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-01-01T09:09:55,269 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2375): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-01-01T09:09:55,276 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1079): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-01-01T09:09:55,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:09:55,282 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:55,284 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:09:55,303 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8fe5994cdb848b90b5ea2670beebb22f, NAME => 'hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:09:55,722 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:55,722 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 8fe5994cdb848b90b5ea2670beebb22f, disabling compactions & flushes 2024-01-01T09:09:55,723 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:55,723 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:55,723 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. after waiting 0 ms 2024-01-01T09:09:55,723 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:55,723 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:55,723 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8fe5994cdb848b90b5ea2670beebb22f: 2024-01-01T09:09:55,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:09:55,739 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1704100195728"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100195728"}]},"ts":"1704100195728"} 2024-01-01T09:09:55,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:09:55,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:09:55,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100195770"}]},"ts":"1704100195770"} 2024-01-01T09:09:55,779 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-01-01T09:09:55,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8fe5994cdb848b90b5ea2670beebb22f, ASSIGN}] 2024-01-01T09:09:55,788 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8fe5994cdb848b90b5ea2670beebb22f, ASSIGN 2024-01-01T09:09:55,790 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8fe5994cdb848b90b5ea2670beebb22f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:09:55,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8fe5994cdb848b90b5ea2670beebb22f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:55,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8fe5994cdb848b90b5ea2670beebb22f, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:09:56,100 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:56,107 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:56,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 8fe5994cdb848b90b5ea2670beebb22f, NAME => 'hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:09:56,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:56,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,109 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,111 INFO [StoreOpener-8fe5994cdb848b90b5ea2670beebb22f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,113 INFO [StoreOpener-8fe5994cdb848b90b5ea2670beebb22f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fe5994cdb848b90b5ea2670beebb22f columnFamilyName info 2024-01-01T09:09:56,113 DEBUG [StoreOpener-8fe5994cdb848b90b5ea2670beebb22f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:56,114 INFO [StoreOpener-8fe5994cdb848b90b5ea2670beebb22f-1 {}] regionserver.HStore(324): Store=8fe5994cdb848b90b5ea2670beebb22f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:09:56,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:09:56,124 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:09:56,125 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 8fe5994cdb848b90b5ea2670beebb22f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69931389, jitterRate=0.042058899998664856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:09:56,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 8fe5994cdb848b90b5ea2670beebb22f: 2024-01-01T09:09:56,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f., pid=6, masterSystemTime=1704100196100 2024-01-01T09:09:56,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:56,132 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:09:56,133 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8fe5994cdb848b90b5ea2670beebb22f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:56,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=6, resume processing ppid=5 2024-01-01T09:09:56,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8fe5994cdb848b90b5ea2670beebb22f, server=jenkins-hbase3.apache.org,44183,1704100133450 in 191 msec 2024-01-01T09:09:56,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=5, resume processing ppid=4 2024-01-01T09:09:56,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8fe5994cdb848b90b5ea2670beebb22f, ASSIGN in 355 msec 2024-01-01T09:09:56,146 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:09:56,147 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100196146"}]},"ts":"1704100196146"} 2024-01-01T09:09:56,151 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-01-01T09:09:56,154 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:09:56,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 885 msec 2024-01-01T09:09:56,181 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-01-01T09:09:56,183 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-01-01T09:09:56,183 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:09:56,183 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:09:56,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1079): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-01-01T09:09:56,242 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-01-01T09:09:56,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 30 msec 2024-01-01T09:09:56,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1079): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-01-01T09:09:56,273 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-01-01T09:09:56,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2024-01-01T09:09:56,287 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-01-01T09:09:56,291 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-01-01T09:09:56,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1195): Master has completed initialization 62.767sec 2024-01-01T09:09:56,294 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-01-01T09:09:56,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-01-01T09:09:56,296 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-01-01T09:09:56,297 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-01-01T09:09:56,297 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-01-01T09:09:56,298 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-01-01T09:09:56,299 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-01-01T09:09:56,308 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1298): Balancer post startup initialization complete, took 0 seconds 2024-01-01T09:09:56,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-01-01T09:09:56,309 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46807,1704100132244-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-01-01T09:09:56,346 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2655594c to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:09:56,355 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd7943b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:09:56,360 DEBUG [Listener at localhost/38993 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-01-01T09:09:56,360 DEBUG [Listener at localhost/38993 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-01-01T09:09:56,374 DEBUG [hconnection-0x186ebdd9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:09:56,390 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35442, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:09:56,402 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:09:56,446 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=196, OpenFileDescriptor=508, MaxFileDescriptor=60000, SystemLoadAverage=85, ProcessCount=173, AvailableMemoryMB=4295 2024-01-01T09:09:56,451 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6d08736f to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:09:56,468 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x6d08736f {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7501f181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:09:56,472 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x29a1918d to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:09:56,476 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64623a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:09:56,485 DEBUG [Listener at localhost/38993 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2024-01-01T09:09:56,488 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-01-01T09:09:56,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:09:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-01-01T09:09:56,504 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:09:56,505 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:56,506 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:09:56,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-01-01T09:09:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-01-01T09:09:56,924 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fb7a1d2a94462bd3dc4f09bfab6db5a4, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:09:56,935 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:56,935 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing fb7a1d2a94462bd3dc4f09bfab6db5a4, disabling compactions & flushes 2024-01-01T09:09:56,935 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:56,935 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:56,935 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. after waiting 0 ms 2024-01-01T09:09:56,935 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:56,936 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:56,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for fb7a1d2a94462bd3dc4f09bfab6db5a4: 2024-01-01T09:09:56,938 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:09:56,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1704100196938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100196938"}]},"ts":"1704100196938"} 2024-01-01T09:09:56,942 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:09:56,943 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:09:56,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100196943"}]},"ts":"1704100196943"} 2024-01-01T09:09:56,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:09:56,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fb7a1d2a94462bd3dc4f09bfab6db5a4, ASSIGN}] 2024-01-01T09:09:56,952 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fb7a1d2a94462bd3dc4f09bfab6db5a4, ASSIGN 2024-01-01T09:09:56,954 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fb7a1d2a94462bd3dc4f09bfab6db5a4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:09:57,105 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fb7a1d2a94462bd3dc4f09bfab6db5a4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:57,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure fb7a1d2a94462bd3dc4f09bfab6db5a4, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:09:57,261 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:57,268 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:57,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7254): Opening region: {ENCODED => fb7a1d2a94462bd3dc4f09bfab6db5a4, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:09:57,269 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,269 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:57,269 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7296): checking encryption for fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,269 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7299): checking classloading for fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,271 INFO [StoreOpener-fb7a1d2a94462bd3dc4f09bfab6db5a4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,273 INFO [StoreOpener-fb7a1d2a94462bd3dc4f09bfab6db5a4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb7a1d2a94462bd3dc4f09bfab6db5a4 columnFamilyName testFamily 2024-01-01T09:09:57,273 DEBUG [StoreOpener-fb7a1d2a94462bd3dc4f09bfab6db5a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:57,274 INFO [StoreOpener-fb7a1d2a94462bd3dc4f09bfab6db5a4-1 {}] regionserver.HStore(324): Store=fb7a1d2a94462bd3dc4f09bfab6db5a4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:09:57,275 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,276 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1076): writing seq id for fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:09:57,283 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:09:57,284 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): Opened fb7a1d2a94462bd3dc4f09bfab6db5a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61961327, jitterRate=-0.07670427858829498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:09:57,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(992): Region open journal for fb7a1d2a94462bd3dc4f09bfab6db5a4: 2024-01-01T09:09:57,287 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2593): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4., pid=11, masterSystemTime=1704100197261 2024-01-01T09:09:57,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2620): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:57,291 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:09:57,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=fb7a1d2a94462bd3dc4f09bfab6db5a4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:57,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=11, resume processing ppid=10 2024-01-01T09:09:57,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure fb7a1d2a94462bd3dc4f09bfab6db5a4, server=jenkins-hbase3.apache.org,44183,1704100133450 in 187 msec 2024-01-01T09:09:57,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=10, resume processing ppid=9 2024-01-01T09:09:57,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=fb7a1d2a94462bd3dc4f09bfab6db5a4, ASSIGN in 348 msec 2024-01-01T09:09:57,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:09:57,303 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100197303"}]},"ts":"1704100197303"} 2024-01-01T09:09:57,306 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:09:57,311 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:09:57,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 811 msec 2024-01-01T09:09:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-01-01T09:09:57,529 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-01-01T09:09:57,529 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:09:57,530 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:09:57,536 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-01-01T09:09:57,536 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:09:57,536 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-01-01T09:09:57,539 DEBUG [hconnection-0x1dbc8593-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:09:57,542 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35454, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:09:57,555 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:09:57,561 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-01-01T09:09:57,578 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:09:57,580 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x6d08736f {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:09:57,587 DEBUG [Listener at localhost/38993 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:09:57,590 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35456, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:09:57,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:09:57,603 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-01-01T09:09:57,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118475 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,605 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-01-01T09:09:57,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118475 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433141 2024-01-01T09:09:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:35456 deadline: 1704100200605, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:09:57,608 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118475 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:09:57,611 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-01-01T09:09:57,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118476 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,612 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-01-01T09:09:57,612 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:09:57,613 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:09:57,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118477 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433139 2024-01-01T09:09:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:35456 deadline: 1704100200613, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:09:57,615 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118477 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:09:57,616 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:09:57,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118478 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6882434854160433138 2024-01-01T09:09:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:35456 deadline: 1704100200616, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:09:57,619 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:09:57,618 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118478 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:09:57,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118479 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:09:57,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6882434854160433137 2024-01-01T09:09:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:35456 deadline: 1704100200618, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:09:57,623 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118479 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:09:57,624 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:09:57,625 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x29a1918d to 127.0.0.1:56255 2024-01-01T09:09:57,626 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:09:57,626 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:09:57,626 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:09:57,634 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:09:57,634 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6d08736f to 127.0.0.1:56255 2024-01-01T09:09:57,648 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=210 (was 196) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x6d08736f java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$483/1654066219.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.lang.Thread.sleep(Native Method) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:600) org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:496) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x29a1918d-SendThread(127.0.0.1:56255) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1235589602_18 at /127.0.0.1:49990 [Waiting for operation #4] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x6d08736f-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: process reaper sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x6d08736f-SendThread(127.0.0.1:56255) sun.nio.ch.Net.shutdown(Native Method) sun.nio.ch.SocketChannelImpl.shutdownOutput(SocketChannelImpl.java:800) sun.nio.ch.SocketAdaptor.shutdownOutput(SocketAdaptor.java:423) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:198) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x29a1918d-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56255@0x29a1918d java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$483/1654066219.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x186ebdd9-shared-pool-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=520 (was 508) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=85 (was 85), ProcessCount=173 (was 173), AvailableMemoryMB=4276 (was 4295) 2024-01-01T09:09:57,658 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=210, OpenFileDescriptor=520, MaxFileDescriptor=60000, SystemLoadAverage=85, ProcessCount=173, AvailableMemoryMB=4275 2024-01-01T09:09:57,661 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x621d0ecb to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:09:57,666 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x621d0ecb {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d59631c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:09:57,667 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x14ec8e9a to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:09:57,672 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8016990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:09:57,675 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:09:57,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-01-01T09:09:57,679 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:09:57,679 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:57,679 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-01-01T09:09:57,680 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:09:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-01-01T09:09:57,694 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 84af1702ea2d10a3f3366f7bcc7c8e30, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 84af1702ea2d10a3f3366f7bcc7c8e30, disabling compactions & flushes 2024-01-01T09:09:58,107 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. after waiting 0 ms 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,107 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,107 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 84af1702ea2d10a3f3366f7bcc7c8e30: 2024-01-01T09:09:58,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:09:58,110 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1704100198110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100198110"}]},"ts":"1704100198110"} 2024-01-01T09:09:58,113 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:09:58,114 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:09:58,114 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100198114"}]},"ts":"1704100198114"} 2024-01-01T09:09:58,116 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:09:58,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=84af1702ea2d10a3f3366f7bcc7c8e30, ASSIGN}] 2024-01-01T09:09:58,122 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=84af1702ea2d10a3f3366f7bcc7c8e30, ASSIGN 2024-01-01T09:09:58,124 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=84af1702ea2d10a3f3366f7bcc7c8e30, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:09:58,275 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=84af1702ea2d10a3f3366f7bcc7c8e30, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:58,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 84af1702ea2d10a3f3366f7bcc7c8e30, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:09:58,431 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:58,436 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7254): Opening region: {ENCODED => 84af1702ea2d10a3f3366f7bcc7c8e30, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:09:58,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:09:58,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7296): checking encryption for 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7299): checking classloading for 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,439 INFO [StoreOpener-84af1702ea2d10a3f3366f7bcc7c8e30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,441 INFO [StoreOpener-84af1702ea2d10a3f3366f7bcc7c8e30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84af1702ea2d10a3f3366f7bcc7c8e30 columnFamilyName testFamily 2024-01-01T09:09:58,441 DEBUG [StoreOpener-84af1702ea2d10a3f3366f7bcc7c8e30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:09:58,442 INFO [StoreOpener-84af1702ea2d10a3f3366f7bcc7c8e30-1 {}] regionserver.HStore(324): Store=84af1702ea2d10a3f3366f7bcc7c8e30/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:09:58,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,448 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1076): writing seq id for 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:09:58,451 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:09:58,452 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1093): Opened 84af1702ea2d10a3f3366f7bcc7c8e30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66144925, jitterRate=-0.014363810420036316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:09:58,453 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(992): Region open journal for 84af1702ea2d10a3f3366f7bcc7c8e30: 2024-01-01T09:09:58,454 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30., pid=14, masterSystemTime=1704100198431 2024-01-01T09:09:58,457 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,457 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:09:58,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=84af1702ea2d10a3f3366f7bcc7c8e30, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:09:58,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=14, resume processing ppid=13 2024-01-01T09:09:58,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 84af1702ea2d10a3f3366f7bcc7c8e30, server=jenkins-hbase3.apache.org,44183,1704100133450 in 183 msec 2024-01-01T09:09:58,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=13, resume processing ppid=12 2024-01-01T09:09:58,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=84af1702ea2d10a3f3366f7bcc7c8e30, ASSIGN in 343 msec 2024-01-01T09:09:58,468 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:09:58,468 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100198468"}]},"ts":"1704100198468"} 2024-01-01T09:09:58,470 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:09:58,474 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:09:58,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 799 msec 2024-01-01T09:09:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-01-01T09:09:58,683 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-01-01T09:09:58,684 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:09:58,684 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:09:58,688 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-01-01T09:09:58,688 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:09:58,688 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-01-01T09:09:58,690 DEBUG [hconnection-0x38fae2a5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:09:58,693 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35470, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:09:58,704 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:09:58,707 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:09:58,708 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:10:00,625 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:00,648 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-01-01T09:10:00,649 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-01-01T09:10:00,650 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-01-01T09:10:00,650 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-01-01T09:10:03,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-01-01T09:10:03,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:03,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-01-01T09:10:03,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:03,151 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-01-01T09:10:03,151 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:03,152 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-01-01T09:10:03,152 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-01-01T09:10:03,152 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-01-01T09:10:03,152 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 6, skipping over this response of 89 bytes 2024-01-01T09:10:05,737 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:10:07,716 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:10:07,717 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-01-01T09:10:07.717Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1979ms, rpcTimeout=1971ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:10:07,719 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2024-01-01T09:10:07,720 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x14ec8e9a to 127.0.0.1:56255 2024-01-01T09:10:07,720 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:07,720 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:07,720 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:07,720 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:07,720 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x621d0ecb to 127.0.0.1:56255 2024-01-01T09:10:07,734 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=203 (was 210), OpenFileDescriptor=518 (was 520), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=80 (was 85), ProcessCount=173 (was 173), AvailableMemoryMB=4249 (was 4275) 2024-01-01T09:10:07,744 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=203, OpenFileDescriptor=518, MaxFileDescriptor=60000, SystemLoadAverage=80, ProcessCount=173, AvailableMemoryMB=4247 2024-01-01T09:10:07,746 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x16b22c0c to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:07,750 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x16b22c0c {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a095599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:07,751 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1d1373a1 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:07,755 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356880fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:07,758 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:07,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-01-01T09:10:07,762 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:07,762 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:07,762 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-01-01T09:10:07,763 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:07,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=15 2024-01-01T09:10:07,776 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 029d4c0e35227b01f80bf9b14d52df3a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 029d4c0e35227b01f80bf9b14d52df3a, disabling compactions & flushes 2024-01-01T09:10:07,788 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. after waiting 0 ms 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:07,788 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:07,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 029d4c0e35227b01f80bf9b14d52df3a: 2024-01-01T09:10:07,790 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:07,790 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1704100207790"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100207790"}]},"ts":"1704100207790"} 2024-01-01T09:10:07,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:07,793 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:07,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100207793"}]},"ts":"1704100207793"} 2024-01-01T09:10:07,795 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:07,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=029d4c0e35227b01f80bf9b14d52df3a, ASSIGN}] 2024-01-01T09:10:07,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=029d4c0e35227b01f80bf9b14d52df3a, ASSIGN 2024-01-01T09:10:07,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=029d4c0e35227b01f80bf9b14d52df3a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:07,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=029d4c0e35227b01f80bf9b14d52df3a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:07,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 029d4c0e35227b01f80bf9b14d52df3a, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:08,110 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:08,116 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:08,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7254): Opening region: {ENCODED => 029d4c0e35227b01f80bf9b14d52df3a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:08,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:08,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7296): checking encryption for 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7299): checking classloading for 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,119 INFO [StoreOpener-029d4c0e35227b01f80bf9b14d52df3a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,120 INFO [StoreOpener-029d4c0e35227b01f80bf9b14d52df3a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 029d4c0e35227b01f80bf9b14d52df3a columnFamilyName testFamily 2024-01-01T09:10:08,121 DEBUG [StoreOpener-029d4c0e35227b01f80bf9b14d52df3a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:08,122 INFO [StoreOpener-029d4c0e35227b01f80bf9b14d52df3a-1 {}] regionserver.HStore(324): Store=029d4c0e35227b01f80bf9b14d52df3a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:08,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1076): writing seq id for 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:10:08,131 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:08,132 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1093): Opened 029d4c0e35227b01f80bf9b14d52df3a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67390457, jitterRate=0.004196062684059143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:08,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(992): Region open journal for 029d4c0e35227b01f80bf9b14d52df3a: 2024-01-01T09:10:08,133 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a., pid=17, masterSystemTime=1704100208110 2024-01-01T09:10:08,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:08,136 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:10:08,136 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=029d4c0e35227b01f80bf9b14d52df3a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:08,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=17, resume processing ppid=16 2024-01-01T09:10:08,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 029d4c0e35227b01f80bf9b14d52df3a, server=jenkins-hbase3.apache.org,44183,1704100133450 in 181 msec 2024-01-01T09:10:08,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=16, resume processing ppid=15 2024-01-01T09:10:08,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=029d4c0e35227b01f80bf9b14d52df3a, ASSIGN in 340 msec 2024-01-01T09:10:08,145 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:08,145 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100208145"}]},"ts":"1704100208145"} 2024-01-01T09:10:08,147 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:08,151 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:08,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 392 msec 2024-01-01T09:10:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=15 2024-01-01T09:10:08,767 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-01-01T09:10:08,767 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:08,767 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:08,770 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:08,771 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:08,771 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-01-01T09:10:08,773 DEBUG [hconnection-0x6fcf0734-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:08,776 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54720, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:08,785 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:08,788 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:08,788 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:10:08,790 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x16b22c0c {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:10:08,790 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x16b22c0c {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:08,792 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:08,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:10:08,796 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:10:08,796 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118491 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:08,797 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:10:08,797 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118491 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:08,798 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-01-01T09:10:12,238 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:12,652 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-01-01T09:10:13,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:13,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-01-01T09:10:13,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:17,806 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2024-01-01T09:10:17,807 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1d1373a1 to 127.0.0.1:56255 2024-01-01T09:10:17,806 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118491 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:10:17,807 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:17,807 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:17,807 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:17,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433125' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3588) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-01-01T09:10:17,807 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:17,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:54732 deadline: 1704100223806, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433125' does not exist 2024-01-01T09:10:17,807 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x16b22c0c to 127.0.0.1:56255 2024-01-01T09:10:17,809 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:17,809 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,44183,1704100133450 for closing scanner id = -6882434854160433125 for 029d4c0e35227b01f80bf9b14d52df3a of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:231) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:239) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-01-01T09:10:17,820 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=204 (was 203) - Thread LEAK? -, OpenFileDescriptor=514 (was 518), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=97 (was 80) - SystemLoadAverage LEAK? -, ProcessCount=173 (was 173), AvailableMemoryMB=4228 (was 4247) 2024-01-01T09:10:17,830 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=204, OpenFileDescriptor=513, MaxFileDescriptor=60000, SystemLoadAverage=97, ProcessCount=173, AvailableMemoryMB=4228 2024-01-01T09:10:17,832 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x658fe78c to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:17,837 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x658fe78c {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b376341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:17,838 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x14ffd654 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:17,844 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be7c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:17,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-01-01T09:10:17,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:17,850 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:17,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-01-01T09:10:17,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-01-01T09:10:18,269 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5c3468183b42b1b15d589cef672ad7b1, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:18,299 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 5c3468183b42b1b15d589cef672ad7b1, disabling compactions & flushes 2024-01-01T09:10:18,680 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. after waiting 0 ms 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:18,680 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:18,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5c3468183b42b1b15d589cef672ad7b1: 2024-01-01T09:10:18,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:18,682 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1704100218682"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100218682"}]},"ts":"1704100218682"} 2024-01-01T09:10:18,685 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:18,686 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:18,686 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100218686"}]},"ts":"1704100218686"} 2024-01-01T09:10:18,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:18,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=5c3468183b42b1b15d589cef672ad7b1, ASSIGN}] 2024-01-01T09:10:18,699 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=5c3468183b42b1b15d589cef672ad7b1, ASSIGN 2024-01-01T09:10:18,700 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=5c3468183b42b1b15d589cef672ad7b1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:18,851 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=5c3468183b42b1b15d589cef672ad7b1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:18,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure 5c3468183b42b1b15d589cef672ad7b1, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-01-01T09:10:19,007 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:19,014 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:19,014 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7254): Opening region: {ENCODED => 5c3468183b42b1b15d589cef672ad7b1, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:19,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:19,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7296): checking encryption for 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7299): checking classloading for 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,017 INFO [StoreOpener-5c3468183b42b1b15d589cef672ad7b1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,019 INFO [StoreOpener-5c3468183b42b1b15d589cef672ad7b1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c3468183b42b1b15d589cef672ad7b1 columnFamilyName testFamily 2024-01-01T09:10:19,019 DEBUG [StoreOpener-5c3468183b42b1b15d589cef672ad7b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:19,020 INFO [StoreOpener-5c3468183b42b1b15d589cef672ad7b1-1 {}] regionserver.HStore(324): Store=5c3468183b42b1b15d589cef672ad7b1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:19,022 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,022 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1076): writing seq id for 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:10:19,029 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:19,030 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): Opened 5c3468183b42b1b15d589cef672ad7b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75106543, jitterRate=0.11917470395565033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:19,031 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(992): Region open journal for 5c3468183b42b1b15d589cef672ad7b1: 2024-01-01T09:10:19,032 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2593): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1., pid=20, masterSystemTime=1704100219007 2024-01-01T09:10:19,035 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2620): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:19,035 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:10:19,035 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=5c3468183b42b1b15d589cef672ad7b1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:19,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=20, resume processing ppid=19 2024-01-01T09:10:19,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure 5c3468183b42b1b15d589cef672ad7b1, server=jenkins-hbase3.apache.org,44183,1704100133450 in 184 msec 2024-01-01T09:10:19,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=19, resume processing ppid=18 2024-01-01T09:10:19,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=5c3468183b42b1b15d589cef672ad7b1, ASSIGN in 342 msec 2024-01-01T09:10:19,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:19,044 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100219044"}]},"ts":"1704100219044"} 2024-01-01T09:10:19,046 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:19,056 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:19,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 1.2090 sec 2024-01-01T09:10:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-01-01T09:10:20,857 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-01-01T09:10:20,857 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:20,857 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:20,863 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:20,864 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:20,864 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-01-01T09:10:20,866 DEBUG [hconnection-0x12fe91f5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:20,869 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48854, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:20,880 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:20,883 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-01-01T09:10:20,884 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-01-01T09:10:20,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118495 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,886 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-01-01T09:10:20,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118495 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433121 2024-01-01T09:10:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48854 deadline: 1704100221886, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:10:20,888 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:20,890 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-01-01T09:10:20,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118496 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,892 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-01-01T09:10:20,892 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:10:20,893 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:10:20,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118497 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433119 2024-01-01T09:10:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48854 deadline: 1704100221894, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:10:20,895 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:20,898 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:10:20,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118498 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6882434854160433118 2024-01-01T09:10:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48854 deadline: 1704100221898, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:10:20,900 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:20,902 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:10:20,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118499 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:20,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6882434854160433117 2024-01-01T09:10:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48854 deadline: 1704100221902, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:10:20,903 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:20,906 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:10:20,906 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x14ffd654 to 127.0.0.1:56255 2024-01-01T09:10:20,906 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:20,906 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:20,906 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:20,906 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:20,907 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x658fe78c to 127.0.0.1:56255 2024-01-01T09:10:20,918 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=208 (was 204) - Thread LEAK? -, OpenFileDescriptor=517 (was 513) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=89 (was 97), ProcessCount=173 (was 173), AvailableMemoryMB=4221 (was 4228) 2024-01-01T09:10:20,928 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=208, OpenFileDescriptor=517, MaxFileDescriptor=60000, SystemLoadAverage=89, ProcessCount=173, AvailableMemoryMB=4220 2024-01-01T09:10:20,930 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x49c419cd to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:20,934 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x49c419cd {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b37fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:20,936 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e78c569 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:20,940 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6386f23b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:20,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-01-01T09:10:20,946 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:20,946 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:20,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-01-01T09:10:20,947 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=21 2024-01-01T09:10:21,360 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1db8a2d69b3d0f098a78503fcb88cb43, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:21,771 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:21,771 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 1db8a2d69b3d0f098a78503fcb88cb43, disabling compactions & flushes 2024-01-01T09:10:21,771 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:21,771 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:21,771 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. after waiting 0 ms 2024-01-01T09:10:21,771 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:21,771 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:21,772 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1db8a2d69b3d0f098a78503fcb88cb43: 2024-01-01T09:10:21,774 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:21,774 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1704100221774"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100221774"}]},"ts":"1704100221774"} 2024-01-01T09:10:21,776 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:21,778 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:21,778 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100221778"}]},"ts":"1704100221778"} 2024-01-01T09:10:21,780 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:21,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=1db8a2d69b3d0f098a78503fcb88cb43, ASSIGN}] 2024-01-01T09:10:21,786 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=1db8a2d69b3d0f098a78503fcb88cb43, ASSIGN 2024-01-01T09:10:21,787 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=1db8a2d69b3d0f098a78503fcb88cb43, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:21,938 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=1db8a2d69b3d0f098a78503fcb88cb43, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:21,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 1db8a2d69b3d0f098a78503fcb88cb43, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=21 2024-01-01T09:10:22,093 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:22,098 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:22,098 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7254): Opening region: {ENCODED => 1db8a2d69b3d0f098a78503fcb88cb43, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:22,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:22,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7296): checking encryption for 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7299): checking classloading for 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,101 INFO [StoreOpener-1db8a2d69b3d0f098a78503fcb88cb43-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,103 INFO [StoreOpener-1db8a2d69b3d0f098a78503fcb88cb43-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db8a2d69b3d0f098a78503fcb88cb43 columnFamilyName testFamily 2024-01-01T09:10:22,104 DEBUG [StoreOpener-1db8a2d69b3d0f098a78503fcb88cb43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:22,105 INFO [StoreOpener-1db8a2d69b3d0f098a78503fcb88cb43-1 {}] regionserver.HStore(324): Store=1db8a2d69b3d0f098a78503fcb88cb43/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:22,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,106 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1076): writing seq id for 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:10:22,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:22,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): Opened 1db8a2d69b3d0f098a78503fcb88cb43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67907174, jitterRate=0.011895745992660522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:22,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(992): Region open journal for 1db8a2d69b3d0f098a78503fcb88cb43: 2024-01-01T09:10:22,115 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43., pid=23, masterSystemTime=1704100222092 2024-01-01T09:10:22,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:22,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:10:22,118 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=1db8a2d69b3d0f098a78503fcb88cb43, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:22,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=23, resume processing ppid=22 2024-01-01T09:10:22,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 1db8a2d69b3d0f098a78503fcb88cb43, server=jenkins-hbase3.apache.org,44183,1704100133450 in 180 msec 2024-01-01T09:10:22,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=22, resume processing ppid=21 2024-01-01T09:10:22,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=1db8a2d69b3d0f098a78503fcb88cb43, ASSIGN in 338 msec 2024-01-01T09:10:22,126 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:22,126 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100222126"}]},"ts":"1704100222126"} 2024-01-01T09:10:22,128 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:22,131 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:22,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 1.1880 sec 2024-01-01T09:10:23,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-01-01T09:10:23,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:23,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-01-01T09:10:23,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:23,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=21 2024-01-01T09:10:23,952 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-01-01T09:10:23,953 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:23,953 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:23,957 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:23,957 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:23,957 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-01-01T09:10:23,959 DEBUG [hconnection-0x22cc07c5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:23,962 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48858, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:23,972 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:23,975 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:23,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:23,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-01-01T09:10:24,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:24,653 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-01-01T09:10:24,654 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-01-01T09:10:25,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 7, skipping over this response of 12 bytes 2024-01-01T09:10:25,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:48858 deadline: 1704100226977, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:26,000 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:26,001 DEBUG [Listener at localhost/38993 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118504 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:10:26,001 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2026 2024-01-01T09:10:26,002 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e78c569 to 127.0.0.1:56255 2024-01-01T09:10:26,002 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:26,002 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:26,002 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:26,002 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:26,002 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x49c419cd to 127.0.0.1:56255 2024-01-01T09:10:26,014 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=209 (was 208) - Thread LEAK? -, OpenFileDescriptor=520 (was 517) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=82 (was 89), ProcessCount=173 (was 173), AvailableMemoryMB=4125 (was 4220) 2024-01-01T09:10:26,024 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=209, OpenFileDescriptor=520, MaxFileDescriptor=60000, SystemLoadAverage=82, ProcessCount=173, AvailableMemoryMB=4124 2024-01-01T09:10:26,026 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ef2d183 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:26,030 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1ef2d183 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b84c9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:26,032 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x279d7ecd to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:26,036 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b690fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:26,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-01-01T09:10:26,042 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:26,042 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:26,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-01-01T09:10:26,043 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-01-01T09:10:26,460 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8b769a5133bcc30546aabb30cc7e069f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 8b769a5133bcc30546aabb30cc7e069f, disabling compactions & flushes 2024-01-01T09:10:26,470 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. after waiting 0 ms 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,470 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,470 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8b769a5133bcc30546aabb30cc7e069f: 2024-01-01T09:10:26,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:26,472 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1704100226471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100226471"}]},"ts":"1704100226471"} 2024-01-01T09:10:26,474 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:26,475 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:26,476 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100226475"}]},"ts":"1704100226475"} 2024-01-01T09:10:26,477 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:26,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=8b769a5133bcc30546aabb30cc7e069f, ASSIGN}] 2024-01-01T09:10:26,487 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=8b769a5133bcc30546aabb30cc7e069f, ASSIGN 2024-01-01T09:10:26,488 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=8b769a5133bcc30546aabb30cc7e069f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:26,639 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=8b769a5133bcc30546aabb30cc7e069f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:26,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure 8b769a5133bcc30546aabb30cc7e069f, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:26,794 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:26,799 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7254): Opening region: {ENCODED => 8b769a5133bcc30546aabb30cc7e069f, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:26,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:26,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7296): checking encryption for 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7299): checking classloading for 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,801 INFO [StoreOpener-8b769a5133bcc30546aabb30cc7e069f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,803 INFO [StoreOpener-8b769a5133bcc30546aabb30cc7e069f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b769a5133bcc30546aabb30cc7e069f columnFamilyName testFamily 2024-01-01T09:10:26,803 DEBUG [StoreOpener-8b769a5133bcc30546aabb30cc7e069f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:26,804 INFO [StoreOpener-8b769a5133bcc30546aabb30cc7e069f-1 {}] regionserver.HStore(324): Store=8b769a5133bcc30546aabb30cc7e069f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:26,805 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,806 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,808 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1076): writing seq id for 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:10:26,811 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:26,811 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): Opened 8b769a5133bcc30546aabb30cc7e069f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72377565, jitterRate=0.07850976288318634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:26,812 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(992): Region open journal for 8b769a5133bcc30546aabb30cc7e069f: 2024-01-01T09:10:26,813 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., pid=26, masterSystemTime=1704100226794 2024-01-01T09:10:26,815 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,815 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:10:26,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=8b769a5133bcc30546aabb30cc7e069f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:26,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=26, resume processing ppid=25 2024-01-01T09:10:26,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure 8b769a5133bcc30546aabb30cc7e069f, server=jenkins-hbase3.apache.org,44183,1704100133450 in 177 msec 2024-01-01T09:10:26,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=25, resume processing ppid=24 2024-01-01T09:10:26,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=8b769a5133bcc30546aabb30cc7e069f, ASSIGN in 334 msec 2024-01-01T09:10:26,823 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:26,824 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100226823"}]},"ts":"1704100226823"} 2024-01-01T09:10:26,825 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:26,828 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:26,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 789 msec 2024-01-01T09:10:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-01-01T09:10:27,046 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-01-01T09:10:27,046 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:27,046 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:27,050 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:27,050 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:27,050 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-01-01T09:10:27,053 DEBUG [hconnection-0x4816f0f5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:27,055 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42920, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:27,066 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:27,069 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:10:27,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:10:28,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 6, skipping over this response of 89 bytes 2024-01-01T09:10:29,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:10:30,076 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:10:30,077 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-01-01T09:10:30.077Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=988ms, rpcTimeout=983ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:10:30,078 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3008 2024-01-01T09:10:30,078 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x279d7ecd to 127.0.0.1:56255 2024-01-01T09:10:30,078 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:30,078 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:30,079 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:30,079 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:30,079 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ef2d183 to 127.0.0.1:56255 2024-01-01T09:10:30,091 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=209 (was 209), OpenFileDescriptor=520 (was 520), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=75 (was 82), ProcessCount=173 (was 173), AvailableMemoryMB=4102 (was 4124) 2024-01-01T09:10:30,101 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=209, OpenFileDescriptor=520, MaxFileDescriptor=60000, SystemLoadAverage=75, ProcessCount=173, AvailableMemoryMB=4101 2024-01-01T09:10:30,103 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0cddbd9f to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:30,110 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x0cddbd9f {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497e0cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:30,112 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1b2a4eae to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:30,117 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee1195f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:30,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-01-01T09:10:30,124 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:30,124 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:30,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-01-01T09:10:30,125 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=27 2024-01-01T09:10:30,538 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 345536859393f3bdc499b7c4b1b8bb53, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:30,589 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:30,654 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 345536859393f3bdc499b7c4b1b8bb53, disabling compactions & flushes 2024-01-01T09:10:30,948 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. after waiting 0 ms 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:30,948 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:30,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 345536859393f3bdc499b7c4b1b8bb53: 2024-01-01T09:10:30,950 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:30,950 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1704100230950"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100230950"}]},"ts":"1704100230950"} 2024-01-01T09:10:30,953 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:30,954 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:30,954 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100230954"}]},"ts":"1704100230954"} 2024-01-01T09:10:30,955 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:30,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=345536859393f3bdc499b7c4b1b8bb53, ASSIGN}] 2024-01-01T09:10:30,967 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=345536859393f3bdc499b7c4b1b8bb53, ASSIGN 2024-01-01T09:10:30,968 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=345536859393f3bdc499b7c4b1b8bb53, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:31,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=345536859393f3bdc499b7c4b1b8bb53, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:31,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 345536859393f3bdc499b7c4b1b8bb53, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=27 2024-01-01T09:10:31,273 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:31,278 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:31,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 345536859393f3bdc499b7c4b1b8bb53, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:31,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:31,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,281 INFO [StoreOpener-345536859393f3bdc499b7c4b1b8bb53-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,284 INFO [StoreOpener-345536859393f3bdc499b7c4b1b8bb53-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 345536859393f3bdc499b7c4b1b8bb53 columnFamilyName testFamily 2024-01-01T09:10:31,284 DEBUG [StoreOpener-345536859393f3bdc499b7c4b1b8bb53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:31,285 INFO [StoreOpener-345536859393f3bdc499b7c4b1b8bb53-1 {}] regionserver.HStore(324): Store=345536859393f3bdc499b7c4b1b8bb53/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:31,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:10:31,293 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:31,294 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 345536859393f3bdc499b7c4b1b8bb53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74290673, jitterRate=0.10701729357242584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:31,295 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 345536859393f3bdc499b7c4b1b8bb53: 2024-01-01T09:10:31,296 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53., pid=29, masterSystemTime=1704100231273 2024-01-01T09:10:31,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:31,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:10:31,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=345536859393f3bdc499b7c4b1b8bb53, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:31,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=29, resume processing ppid=28 2024-01-01T09:10:31,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 345536859393f3bdc499b7c4b1b8bb53, server=jenkins-hbase3.apache.org,44183,1704100133450 in 180 msec 2024-01-01T09:10:31,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=28, resume processing ppid=27 2024-01-01T09:10:31,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=345536859393f3bdc499b7c4b1b8bb53, ASSIGN in 339 msec 2024-01-01T09:10:31,307 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:31,307 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100231307"}]},"ts":"1704100231307"} 2024-01-01T09:10:31,309 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:31,313 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:31,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 1.1930 sec 2024-01-01T09:10:32,302 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=27 2024-01-01T09:10:33,129 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-01-01T09:10:33,129 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:33,130 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:33,133 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:33,134 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:33,134 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-01-01T09:10:33,136 DEBUG [hconnection-0x142942d8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:33,139 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42936, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:33,148 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:33,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-01-01T09:10:33,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:33,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-01-01T09:10:33,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:33,150 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:33,151 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:10:33,152 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x0cddbd9f {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:10:33,152 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x0cddbd9f {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:33,154 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42950, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:10:33,158 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:10:33,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118516 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:33,160 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:10:33,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118516 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:33,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-01-01T09:10:36,167 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3007 2024-01-01T09:10:36,167 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1b2a4eae to 127.0.0.1:56255 2024-01-01T09:10:36,167 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118516 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:10:36,167 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:36,167 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:36,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433100' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3588) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-01-01T09:10:36,168 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:36,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:42950 deadline: 1704100237167, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433100' does not exist 2024-01-01T09:10:36,168 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:36,168 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0cddbd9f to 127.0.0.1:56255 2024-01-01T09:10:36,168 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,44183,1704100133450 for closing scanner id = -6882434854160433100 for 345536859393f3bdc499b7c4b1b8bb53 of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:231) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:239) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-01-01T09:10:36,183 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=210 (was 209) - Thread LEAK? -, OpenFileDescriptor=518 (was 520), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=69 (was 75), ProcessCount=173 (was 173), AvailableMemoryMB=4098 (was 4101) 2024-01-01T09:10:36,193 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=210, OpenFileDescriptor=518, MaxFileDescriptor=60000, SystemLoadAverage=69, ProcessCount=173, AvailableMemoryMB=4097 2024-01-01T09:10:36,195 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x22428006 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:36,205 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x22428006 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c51f57a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:36,206 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4a5391e3 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:36,210 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4d76ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:36,213 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:36,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-01-01T09:10:36,215 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:36,216 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:36,216 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-01-01T09:10:36,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-01-01T09:10:36,329 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-01-01T09:10:36,329 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-01-01T09:10:36,630 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1c9c1fffb4d91d95f7adedde98457b0a, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:36,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:36,642 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 1c9c1fffb4d91d95f7adedde98457b0a, disabling compactions & flushes 2024-01-01T09:10:36,643 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. after waiting 0 ms 2024-01-01T09:10:36,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,643 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,643 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1c9c1fffb4d91d95f7adedde98457b0a: 2024-01-01T09:10:36,646 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:36,646 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1704100236646"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100236646"}]},"ts":"1704100236646"} 2024-01-01T09:10:36,649 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:36,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:36,651 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100236651"}]},"ts":"1704100236651"} 2024-01-01T09:10:36,654 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:36,655 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-01-01T09:10:36,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=1c9c1fffb4d91d95f7adedde98457b0a, ASSIGN}] 2024-01-01T09:10:36,660 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=1c9c1fffb4d91d95f7adedde98457b0a, ASSIGN 2024-01-01T09:10:36,661 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:36,666 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=1c9c1fffb4d91d95f7adedde98457b0a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:36,817 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=1c9c1fffb4d91d95f7adedde98457b0a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:36,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 1c9c1fffb4d91d95f7adedde98457b0a, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:36,973 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:36,979 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,979 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7254): Opening region: {ENCODED => 1c9c1fffb4d91d95f7adedde98457b0a, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:36,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:36,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7296): checking encryption for 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7299): checking classloading for 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,982 INFO [StoreOpener-1c9c1fffb4d91d95f7adedde98457b0a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,984 INFO [StoreOpener-1c9c1fffb4d91d95f7adedde98457b0a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c9c1fffb4d91d95f7adedde98457b0a columnFamilyName testFamily 2024-01-01T09:10:36,984 DEBUG [StoreOpener-1c9c1fffb4d91d95f7adedde98457b0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:36,985 INFO [StoreOpener-1c9c1fffb4d91d95f7adedde98457b0a-1 {}] regionserver.HStore(324): Store=1c9c1fffb4d91d95f7adedde98457b0a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:36,986 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,987 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,990 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1076): writing seq id for 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:10:36,992 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:36,993 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1093): Opened 1c9c1fffb4d91d95f7adedde98457b0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73604122, jitterRate=0.09678688645362854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:36,994 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(992): Region open journal for 1c9c1fffb4d91d95f7adedde98457b0a: 2024-01-01T09:10:36,995 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a., pid=32, masterSystemTime=1704100236973 2024-01-01T09:10:36,997 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,997 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:10:36,998 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=1c9c1fffb4d91d95f7adedde98457b0a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:37,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=32, resume processing ppid=31 2024-01-01T09:10:37,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 1c9c1fffb4d91d95f7adedde98457b0a, server=jenkins-hbase3.apache.org,44183,1704100133450 in 180 msec 2024-01-01T09:10:37,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=31, resume processing ppid=30 2024-01-01T09:10:37,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=1c9c1fffb4d91d95f7adedde98457b0a, ASSIGN in 343 msec 2024-01-01T09:10:37,005 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:37,006 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100237005"}]},"ts":"1704100237005"} 2024-01-01T09:10:37,008 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:37,012 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:37,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 799 msec 2024-01-01T09:10:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-01-01T09:10:37,220 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-01-01T09:10:37,220 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:37,220 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:37,224 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:37,225 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:37,225 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-01-01T09:10:37,227 DEBUG [hconnection-0x52155c97-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:37,229 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44716, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:37,238 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:37,241 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:37,243 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118520 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:10:37,243 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-01-01T09:10:42,482 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:42,657 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-01-01T09:10:43,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-01-01T09:10:43,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:43,744 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-01-01T09:10:43,744 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4a5391e3 to 127.0.0.1:56255 2024-01-01T09:10:43,744 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:43,744 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:43,745 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:43,745 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:43,745 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x22428006 to 127.0.0.1:56255 2024-01-01T09:10:43,758 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=211 (was 210) - Thread LEAK? -, OpenFileDescriptor=515 (was 518), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=64 (was 69), ProcessCount=173 (was 173), AvailableMemoryMB=4093 (was 4097) 2024-01-01T09:10:43,768 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=211, OpenFileDescriptor=515, MaxFileDescriptor=60000, SystemLoadAverage=64, ProcessCount=173, AvailableMemoryMB=4093 2024-01-01T09:10:43,769 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x288d9743 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:43,775 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x288d9743 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754facaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:43,776 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2bf33bd5 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:43,780 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a536db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:43,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-01-01T09:10:43,785 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:43,785 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:43,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-01-01T09:10:43,786 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-01-01T09:10:44,197 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b12eec449640ec3fe233eaeeed122249, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing b12eec449640ec3fe233eaeeed122249, disabling compactions & flushes 2024-01-01T09:10:44,607 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. after waiting 0 ms 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,607 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,607 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for b12eec449640ec3fe233eaeeed122249: 2024-01-01T09:10:44,609 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:44,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1704100244609"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100244609"}]},"ts":"1704100244609"} 2024-01-01T09:10:44,612 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:44,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:44,613 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100244613"}]},"ts":"1704100244613"} 2024-01-01T09:10:44,615 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:44,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b12eec449640ec3fe233eaeeed122249, ASSIGN}] 2024-01-01T09:10:44,623 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b12eec449640ec3fe233eaeeed122249, ASSIGN 2024-01-01T09:10:44,624 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b12eec449640ec3fe233eaeeed122249, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:44,775 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=b12eec449640ec3fe233eaeeed122249, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:44,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure b12eec449640ec3fe233eaeeed122249, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-01-01T09:10:44,930 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:44,935 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,936 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7254): Opening region: {ENCODED => b12eec449640ec3fe233eaeeed122249, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:44,936 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,936 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:44,936 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7296): checking encryption for b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,936 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7299): checking classloading for b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,938 INFO [StoreOpener-b12eec449640ec3fe233eaeeed122249-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,940 INFO [StoreOpener-b12eec449640ec3fe233eaeeed122249-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b12eec449640ec3fe233eaeeed122249 columnFamilyName testFamily 2024-01-01T09:10:44,940 DEBUG [StoreOpener-b12eec449640ec3fe233eaeeed122249-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:44,940 INFO [StoreOpener-b12eec449640ec3fe233eaeeed122249-1 {}] regionserver.HStore(324): Store=b12eec449640ec3fe233eaeeed122249/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:44,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,942 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,945 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1076): writing seq id for b12eec449640ec3fe233eaeeed122249 2024-01-01T09:10:44,948 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:44,948 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): Opened b12eec449640ec3fe233eaeeed122249; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61473442, jitterRate=-0.08397433161735535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:44,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(992): Region open journal for b12eec449640ec3fe233eaeeed122249: 2024-01-01T09:10:44,950 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249., pid=35, masterSystemTime=1704100244930 2024-01-01T09:10:44,952 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,952 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:10:44,953 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=b12eec449640ec3fe233eaeeed122249, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:44,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=35, resume processing ppid=34 2024-01-01T09:10:44,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure b12eec449640ec3fe233eaeeed122249, server=jenkins-hbase3.apache.org,44183,1704100133450 in 177 msec 2024-01-01T09:10:44,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=34, resume processing ppid=33 2024-01-01T09:10:44,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=b12eec449640ec3fe233eaeeed122249, ASSIGN in 337 msec 2024-01-01T09:10:44,961 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:44,962 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100244961"}]},"ts":"1704100244961"} 2024-01-01T09:10:44,963 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:44,968 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:44,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 1.1850 sec 2024-01-01T09:10:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-01-01T09:10:46,791 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-01-01T09:10:46,791 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:46,791 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:46,795 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:46,796 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:46,796 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-01-01T09:10:46,798 DEBUG [hconnection-0x3830a6fb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:46,801 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35068, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:46,810 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:46,812 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:46,814 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118524 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:10:46,815 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-01-01T09:10:48,315 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1503 2024-01-01T09:10:48,316 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2bf33bd5 to 127.0.0.1:56255 2024-01-01T09:10:48,316 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:48,316 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:48,316 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:48,316 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:48,316 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x288d9743 to 127.0.0.1:56255 2024-01-01T09:10:48,330 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=214 (was 211) - Thread LEAK? -, OpenFileDescriptor=517 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=59 (was 64), ProcessCount=171 (was 173), AvailableMemoryMB=5459 (was 4093) - AvailableMemoryMB LEAK? - 2024-01-01T09:10:48,339 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=214, OpenFileDescriptor=517, MaxFileDescriptor=60000, SystemLoadAverage=59, ProcessCount=171, AvailableMemoryMB=5459 2024-01-01T09:10:48,341 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3b411cdd to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:48,351 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x3b411cdd {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788f47e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:48,352 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x77e30855 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:48,356 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683bbc1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:48,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-01-01T09:10:48,361 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:48,361 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:48,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-01-01T09:10:48,362 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2024-01-01T09:10:48,376 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2cb844b5df2310f6a3c3ae750e9d60fb, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:48,658 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-01-01T09:10:48,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:48,787 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 2cb844b5df2310f6a3c3ae750e9d60fb, disabling compactions & flushes 2024-01-01T09:10:48,787 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:48,787 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:48,787 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. after waiting 0 ms 2024-01-01T09:10:48,787 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:48,787 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:48,787 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2cb844b5df2310f6a3c3ae750e9d60fb: 2024-01-01T09:10:48,788 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:48,789 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1704100248789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100248789"}]},"ts":"1704100248789"} 2024-01-01T09:10:48,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:48,792 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:48,793 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100248792"}]},"ts":"1704100248792"} 2024-01-01T09:10:48,794 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:48,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2cb844b5df2310f6a3c3ae750e9d60fb, ASSIGN}] 2024-01-01T09:10:48,801 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2cb844b5df2310f6a3c3ae750e9d60fb, ASSIGN 2024-01-01T09:10:48,802 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2cb844b5df2310f6a3c3ae750e9d60fb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:48,953 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=2cb844b5df2310f6a3c3ae750e9d60fb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:48,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 2cb844b5df2310f6a3c3ae750e9d60fb, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:49,108 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:49,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:49,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7254): Opening region: {ENCODED => 2cb844b5df2310f6a3c3ae750e9d60fb, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:49,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:49,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7296): checking encryption for 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7299): checking classloading for 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,115 INFO [StoreOpener-2cb844b5df2310f6a3c3ae750e9d60fb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,117 INFO [StoreOpener-2cb844b5df2310f6a3c3ae750e9d60fb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cb844b5df2310f6a3c3ae750e9d60fb columnFamilyName testFamily 2024-01-01T09:10:49,117 DEBUG [StoreOpener-2cb844b5df2310f6a3c3ae750e9d60fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:49,117 INFO [StoreOpener-2cb844b5df2310f6a3c3ae750e9d60fb-1 {}] regionserver.HStore(324): Store=2cb844b5df2310f6a3c3ae750e9d60fb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:49,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1076): writing seq id for 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:10:49,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:49,125 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): Opened 2cb844b5df2310f6a3c3ae750e9d60fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66260904, jitterRate=-0.012635588645935059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:49,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(992): Region open journal for 2cb844b5df2310f6a3c3ae750e9d60fb: 2024-01-01T09:10:49,127 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb., pid=38, masterSystemTime=1704100249108 2024-01-01T09:10:49,129 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:49,129 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:10:49,130 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=2cb844b5df2310f6a3c3ae750e9d60fb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:49,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=38, resume processing ppid=37 2024-01-01T09:10:49,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 2cb844b5df2310f6a3c3ae750e9d60fb, server=jenkins-hbase3.apache.org,44183,1704100133450 in 176 msec 2024-01-01T09:10:49,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=37, resume processing ppid=36 2024-01-01T09:10:49,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=2cb844b5df2310f6a3c3ae750e9d60fb, ASSIGN in 334 msec 2024-01-01T09:10:49,137 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:49,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100249137"}]},"ts":"1704100249137"} 2024-01-01T09:10:49,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:49,142 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:49,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 784 msec 2024-01-01T09:10:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2024-01-01T09:10:49,365 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-01-01T09:10:49,365 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:49,365 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:49,369 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:49,369 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:49,369 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-01-01T09:10:49,371 DEBUG [hconnection-0x4f47737c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:49,374 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35074, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:49,382 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:49,385 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57dfabe0 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:49,390 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a467e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:49,391 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:49,393 DEBUG [hconnection-0x247c4dcc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:49,395 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35082, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:49,399 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:10:49,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118529 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:49,400 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:10:49,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118529 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:10:49,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-01-01T09:10:50,407 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-01-01T09:10:50.407Z, RpcRetryingCaller{globalStartTime=2024-01-01T09:10:49.401Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-01-01T09:10:50.407Z, RpcRetryingCaller{globalStartTime=2024-01-01T09:10:49.401Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:10:50,407 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1007 2024-01-01T09:10:50,408 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x57dfabe0 to 127.0.0.1:56255 2024-01-01T09:10:50,408 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:50,408 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x77e30855 to 127.0.0.1:56255 2024-01-01T09:10:50,408 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:50,408 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:50,409 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:50,409 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:50,409 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3b411cdd to 127.0.0.1:56255 2024-01-01T09:10:50,421 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=218 (was 214) - Thread LEAK? -, OpenFileDescriptor=525 (was 517) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=62 (was 59) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=5458 (was 5459) 2024-01-01T09:10:50,432 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=218, OpenFileDescriptor=524, MaxFileDescriptor=60000, SystemLoadAverage=62, ProcessCount=171, AvailableMemoryMB=5458 2024-01-01T09:10:50,434 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3bb7c00e to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:50,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:50,439 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x3bb7c00e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be97587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:50,440 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x51277021 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:50,445 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244a61e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:50,447 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:50,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-01-01T09:10:50,450 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:50,450 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:50,451 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-01-01T09:10:50,451 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2024-01-01T09:10:50,471 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9a4fa884b1f0e010faf65e75081899ff, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 9a4fa884b1f0e010faf65e75081899ff, disabling compactions & flushes 2024-01-01T09:10:50,882 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. after waiting 0 ms 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:50,882 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:50,882 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9a4fa884b1f0e010faf65e75081899ff: 2024-01-01T09:10:50,883 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:50,884 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1704100250884"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100250884"}]},"ts":"1704100250884"} 2024-01-01T09:10:50,886 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:50,887 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:50,887 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100250887"}]},"ts":"1704100250887"} 2024-01-01T09:10:50,889 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:50,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=9a4fa884b1f0e010faf65e75081899ff, ASSIGN}] 2024-01-01T09:10:50,894 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=9a4fa884b1f0e010faf65e75081899ff, ASSIGN 2024-01-01T09:10:50,895 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=9a4fa884b1f0e010faf65e75081899ff, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:50,902 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:10:51,046 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=9a4fa884b1f0e010faf65e75081899ff, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:51,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 9a4fa884b1f0e010faf65e75081899ff, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:51,200 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:51,205 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:51,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => 9a4fa884b1f0e010faf65e75081899ff, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:51,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:51,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,207 INFO [StoreOpener-9a4fa884b1f0e010faf65e75081899ff-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,209 INFO [StoreOpener-9a4fa884b1f0e010faf65e75081899ff-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a4fa884b1f0e010faf65e75081899ff columnFamilyName testFamily 2024-01-01T09:10:51,209 DEBUG [StoreOpener-9a4fa884b1f0e010faf65e75081899ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:51,209 INFO [StoreOpener-9a4fa884b1f0e010faf65e75081899ff-1 {}] regionserver.HStore(324): Store=9a4fa884b1f0e010faf65e75081899ff/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:51,210 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,211 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,213 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:10:51,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:51,217 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1093): Opened 9a4fa884b1f0e010faf65e75081899ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73446771, jitterRate=0.09444217383861542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:51,217 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(992): Region open journal for 9a4fa884b1f0e010faf65e75081899ff: 2024-01-01T09:10:51,218 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff., pid=41, masterSystemTime=1704100251200 2024-01-01T09:10:51,220 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:51,220 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:10:51,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=9a4fa884b1f0e010faf65e75081899ff, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:51,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=41, resume processing ppid=40 2024-01-01T09:10:51,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 9a4fa884b1f0e010faf65e75081899ff, server=jenkins-hbase3.apache.org,44183,1704100133450 in 175 msec 2024-01-01T09:10:51,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=40, resume processing ppid=39 2024-01-01T09:10:51,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=9a4fa884b1f0e010faf65e75081899ff, ASSIGN in 332 msec 2024-01-01T09:10:51,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:51,228 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100251228"}]},"ts":"1704100251228"} 2024-01-01T09:10:51,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:51,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:51,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 786 msec 2024-01-01T09:10:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2024-01-01T09:10:51,454 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-01-01T09:10:51,455 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:10:51,455 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:51,459 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-01-01T09:10:51,459 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:10:51,459 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-01-01T09:10:51,461 DEBUG [hconnection-0xd12010c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:10:51,463 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35086, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:10:51,471 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:10:51,474 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:10:51,475 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:51,476 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-01-01T09:10:53,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-01-01T09:10:53,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:53,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-01-01T09:10:53,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:53,151 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-01-01T09:10:53,151 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:10:54,658 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-01-01T09:10:54,659 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-01-01T09:10:55,252 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-01-01T09:10:55,258 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8fe5994cdb848b90b5ea2670beebb22f changed from -1.0 to 0.0, refreshing cache 2024-01-01T09:10:56,707 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:10:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 7, skipping over this response of 12 bytes 2024-01-01T09:10:58,492 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:58,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.31.12.81:35086 deadline: 1704100260476, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:10:58,493 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:10:58,494 DEBUG [Listener at localhost/38993 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118533 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:10:58,495 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7021 2024-01-01T09:10:58,495 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x51277021 to 127.0.0.1:56255 2024-01-01T09:10:58,496 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:58,496 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:10:58,496 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:10:58,496 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:10:58,496 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3bb7c00e to 127.0.0.1:56255 2024-01-01T09:10:58,507 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=214 (was 218), OpenFileDescriptor=513 (was 524), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=56 (was 62), ProcessCount=171 (was 171), AvailableMemoryMB=5451 (was 5458) 2024-01-01T09:10:58,517 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=214, OpenFileDescriptor=513, MaxFileDescriptor=60000, SystemLoadAverage=56, ProcessCount=171, AvailableMemoryMB=5451 2024-01-01T09:10:58,518 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1836de00 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:58,524 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1836de00 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c5039a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:58,525 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4ef90853 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:10:58,528 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f373576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:10:58,531 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:10:58,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-01-01T09:10:58,533 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:10:58,533 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:58,534 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-01-01T09:10:58,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:10:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-01-01T09:10:58,946 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c15775553a47462f4ada40853ab24526, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:10:59,357 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:59,357 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing c15775553a47462f4ada40853ab24526, disabling compactions & flushes 2024-01-01T09:10:59,357 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,357 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,357 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. after waiting 0 ms 2024-01-01T09:10:59,357 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,357 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,358 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for c15775553a47462f4ada40853ab24526: 2024-01-01T09:10:59,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:10:59,359 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1704100259359"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100259359"}]},"ts":"1704100259359"} 2024-01-01T09:10:59,361 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:10:59,362 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:10:59,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100259362"}]},"ts":"1704100259362"} 2024-01-01T09:10:59,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:10:59,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c15775553a47462f4ada40853ab24526, ASSIGN}] 2024-01-01T09:10:59,370 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c15775553a47462f4ada40853ab24526, ASSIGN 2024-01-01T09:10:59,371 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c15775553a47462f4ada40853ab24526, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:10:59,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=c15775553a47462f4ada40853ab24526, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:59,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure c15775553a47462f4ada40853ab24526, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:10:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-01-01T09:10:59,676 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:59,681 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7254): Opening region: {ENCODED => c15775553a47462f4ada40853ab24526, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:10:59,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:10:59,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7296): checking encryption for c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7299): checking classloading for c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,683 INFO [StoreOpener-c15775553a47462f4ada40853ab24526-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,685 INFO [StoreOpener-c15775553a47462f4ada40853ab24526-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c15775553a47462f4ada40853ab24526 columnFamilyName testFamily 2024-01-01T09:10:59,685 DEBUG [StoreOpener-c15775553a47462f4ada40853ab24526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:10:59,686 INFO [StoreOpener-c15775553a47462f4ada40853ab24526-1 {}] regionserver.HStore(324): Store=c15775553a47462f4ada40853ab24526/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:10:59,687 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,687 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1076): writing seq id for c15775553a47462f4ada40853ab24526 2024-01-01T09:10:59,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:10:59,694 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1093): Opened c15775553a47462f4ada40853ab24526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63336424, jitterRate=-0.05621373653411865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:10:59,695 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(992): Region open journal for c15775553a47462f4ada40853ab24526: 2024-01-01T09:10:59,696 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., pid=44, masterSystemTime=1704100259676 2024-01-01T09:10:59,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,698 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:10:59,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=c15775553a47462f4ada40853ab24526, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:10:59,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=44, resume processing ppid=43 2024-01-01T09:10:59,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure c15775553a47462f4ada40853ab24526, server=jenkins-hbase3.apache.org,44183,1704100133450 in 176 msec 2024-01-01T09:10:59,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=43, resume processing ppid=42 2024-01-01T09:10:59,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=c15775553a47462f4ada40853ab24526, ASSIGN in 333 msec 2024-01-01T09:10:59,705 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:10:59,706 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100259705"}]},"ts":"1704100259705"} 2024-01-01T09:10:59,708 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:10:59,712 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:10:59,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 1.1810 sec 2024-01-01T09:11:00,660 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-01-01T09:11:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-01-01T09:11:01,539 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-01-01T09:11:01,539 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:11:01,539 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:01,543 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-01-01T09:11:01,543 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:01,543 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-01-01T09:11:01,545 DEBUG [hconnection-0x47281649-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:01,547 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46164, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:01,557 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:01,559 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:11:01,560 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:11:01,561 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1836de00 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:11:01,561 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1836de00 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:01,562 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:01,566 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:11:03,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-01-01T09:11:03,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:05,184 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:05,236 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433132 expired clientIPAndPort=172.31.12.81:35470, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:11:07,577 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6011ms, rpcTimeout=6000ms 2024-01-01T09:11:07,577 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6011ms, rpcTimeout=6000ms 2024-01-01T09:11:07,577 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:11:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 1, skipping over this response of 89 bytes 2024-01-01T09:11:08,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:08,594 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:11:10,566 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1966ms 2024-01-01T09:11:10,566 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1966ms 2024-01-01T09:11:10,566 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:11:10,567 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-01-01T09:11:07.578Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6011ms, rpcTimeout=6000ms 2024-01-01T09:11:10.566Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1966ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1973ms, rpcTimeout=1966ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-01-01T09:11:10,569 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9009 2024-01-01T09:11:10,569 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4ef90853 to 127.0.0.1:56255 2024-01-01T09:11:10,570 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:10,570 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:10,570 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:10,570 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:10,570 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1836de00 to 127.0.0.1:56255 2024-01-01T09:11:10,582 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=212 (was 214), OpenFileDescriptor=506 (was 513), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=44 (was 56), ProcessCount=171 (was 171), AvailableMemoryMB=5433 (was 5451) 2024-01-01T09:11:10,592 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=212, OpenFileDescriptor=506, MaxFileDescriptor=60000, SystemLoadAverage=44, ProcessCount=171, AvailableMemoryMB=5432 2024-01-01T09:11:10,593 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3b9d16d9 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:10,598 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x3b9d16d9 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aeba635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:10,598 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x72eea463 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:10,604 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0a5c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:10,606 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:10,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-01-01T09:11:10,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:10,609 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:10,609 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-01-01T09:11:10,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2024-01-01T09:11:11,022 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ca7ad7cffa9e545916841f9a00e288f3, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing ca7ad7cffa9e545916841f9a00e288f3, disabling compactions & flushes 2024-01-01T09:11:11,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. after waiting 0 ms 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for ca7ad7cffa9e545916841f9a00e288f3: 2024-01-01T09:11:11,434 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:11,434 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1704100271434"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100271434"}]},"ts":"1704100271434"} 2024-01-01T09:11:11,436 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:11,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:11,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100271437"}]},"ts":"1704100271437"} 2024-01-01T09:11:11,438 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:11:11,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=ca7ad7cffa9e545916841f9a00e288f3, ASSIGN}] 2024-01-01T09:11:11,444 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=ca7ad7cffa9e545916841f9a00e288f3, ASSIGN 2024-01-01T09:11:11,445 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=ca7ad7cffa9e545916841f9a00e288f3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:11,595 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=ca7ad7cffa9e545916841f9a00e288f3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:11,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure ca7ad7cffa9e545916841f9a00e288f3, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2024-01-01T09:11:11,749 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:11,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7254): Opening region: {ENCODED => ca7ad7cffa9e545916841f9a00e288f3, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:11,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:11,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7296): checking encryption for ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7299): checking classloading for ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,759 INFO [StoreOpener-ca7ad7cffa9e545916841f9a00e288f3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,760 INFO [StoreOpener-ca7ad7cffa9e545916841f9a00e288f3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca7ad7cffa9e545916841f9a00e288f3 columnFamilyName testFamily 2024-01-01T09:11:11,760 DEBUG [StoreOpener-ca7ad7cffa9e545916841f9a00e288f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:11,761 INFO [StoreOpener-ca7ad7cffa9e545916841f9a00e288f3-1 {}] regionserver.HStore(324): Store=ca7ad7cffa9e545916841f9a00e288f3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:11,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1076): writing seq id for ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:11:11,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:11,769 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1093): Opened ca7ad7cffa9e545916841f9a00e288f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73600361, jitterRate=0.09673084318637848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:11,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(992): Region open journal for ca7ad7cffa9e545916841f9a00e288f3: 2024-01-01T09:11:11,770 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3., pid=47, masterSystemTime=1704100271749 2024-01-01T09:11:11,772 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,772 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:11:11,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=ca7ad7cffa9e545916841f9a00e288f3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:11,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=47, resume processing ppid=46 2024-01-01T09:11:11,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure ca7ad7cffa9e545916841f9a00e288f3, server=jenkins-hbase3.apache.org,44183,1704100133450 in 178 msec 2024-01-01T09:11:11,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=46, resume processing ppid=45 2024-01-01T09:11:11,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=ca7ad7cffa9e545916841f9a00e288f3, ASSIGN in 334 msec 2024-01-01T09:11:11,779 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:11,779 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100271779"}]},"ts":"1704100271779"} 2024-01-01T09:11:11,781 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:11:11,784 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:11,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.1780 sec 2024-01-01T09:11:12,241 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433130 expired clientIPAndPort=172.31.12.81:35470, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:11:12,662 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-01-01T09:11:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-01-01T09:11:13,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2024-01-01T09:11:13,614 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-01-01T09:11:13,614 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:11:13,615 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:13,618 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-01-01T09:11:13,618 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:13,618 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-01-01T09:11:13,620 DEBUG [hconnection-0x64d1029d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:13,622 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35192, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:13,630 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:13,634 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3b05ddcf to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:13,639 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c208b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:13,639 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:11:13,641 DEBUG [hconnection-0x143341ef-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:13,642 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35198, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:13,645 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:11:13,647 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118546 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:13,648 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:11:13,649 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118546 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:13,649 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-01-01T09:11:15,095 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:11:17,303 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:18,345 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433125 expired clientIPAndPort=172.31.12.81:54732, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:11:19,657 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-01-01T09:11:19.657Z, RpcRetryingCaller{globalStartTime=2024-01-01T09:11:13.648Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-01-01T09:11:19.657Z, RpcRetryingCaller{globalStartTime=2024-01-01T09:11:13.648Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6008ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:11:19,658 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6009 2024-01-01T09:11:19,658 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3b05ddcf to 127.0.0.1:56255 2024-01-01T09:11:19,658 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:19,658 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x72eea463 to 127.0.0.1:56255 2024-01-01T09:11:19,658 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:19,659 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:19,659 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:19,659 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:19,659 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3b9d16d9 to 127.0.0.1:56255 2024-01-01T09:11:19,671 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=217 (was 212) - Thread LEAK? -, OpenFileDescriptor=514 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=37 (was 44), ProcessCount=171 (was 171), AvailableMemoryMB=5415 (was 5432) 2024-01-01T09:11:19,681 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=217, OpenFileDescriptor=514, MaxFileDescriptor=60000, SystemLoadAverage=37, ProcessCount=171, AvailableMemoryMB=5415 2024-01-01T09:11:19,682 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ab6ab10 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:19,686 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1ab6ab10 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297c0786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:19,687 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7d8e303d to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:19,692 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f70345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:19,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-01-01T09:11:19,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:19,697 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:19,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-01-01T09:11:19,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-01-01T09:11:20,111 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 257e5f4c014f577810ee0791199ada41, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:20,150 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1668): Closing 257e5f4c014f577810ee0791199ada41, disabling compactions & flushes 2024-01-01T09:11:20,522 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. after waiting 0 ms 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,522 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,522 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1622): Region close journal for 257e5f4c014f577810ee0791199ada41: 2024-01-01T09:11:20,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:20,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1704100280523"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100280523"}]},"ts":"1704100280523"} 2024-01-01T09:11:20,526 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:20,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:20,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100280526"}]},"ts":"1704100280526"} 2024-01-01T09:11:20,528 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-01-01T09:11:20,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=257e5f4c014f577810ee0791199ada41, ASSIGN}] 2024-01-01T09:11:20,533 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=257e5f4c014f577810ee0791199ada41, ASSIGN 2024-01-01T09:11:20,534 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=257e5f4c014f577810ee0791199ada41, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:20,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=257e5f4c014f577810ee0791199ada41, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:20,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure 257e5f4c014f577810ee0791199ada41, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-01-01T09:11:20,839 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:20,844 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7254): Opening region: {ENCODED => 257e5f4c014f577810ee0791199ada41, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false 257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7296): checking encryption for 257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7299): checking classloading for 257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,847 INFO [StoreOpener-257e5f4c014f577810ee0791199ada41-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,848 INFO [StoreOpener-257e5f4c014f577810ee0791199ada41-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257e5f4c014f577810ee0791199ada41 columnFamilyName testFamily 2024-01-01T09:11:20,849 DEBUG [StoreOpener-257e5f4c014f577810ee0791199ada41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:20,849 INFO [StoreOpener-257e5f4c014f577810ee0791199ada41-1 {}] regionserver.HStore(324): Store=257e5f4c014f577810ee0791199ada41/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:20,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,853 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1076): writing seq id for 257e5f4c014f577810ee0791199ada41 2024-01-01T09:11:20,855 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:20,856 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): Opened 257e5f4c014f577810ee0791199ada41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73165172, jitterRate=0.09024602174758911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:20,856 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(992): Region open journal for 257e5f4c014f577810ee0791199ada41: 2024-01-01T09:11:20,857 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., pid=50, masterSystemTime=1704100280839 2024-01-01T09:11:20,859 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,859 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:11:20,860 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=257e5f4c014f577810ee0791199ada41, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:20,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=50, resume processing ppid=49 2024-01-01T09:11:20,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure 257e5f4c014f577810ee0791199ada41, server=jenkins-hbase3.apache.org,44183,1704100133450 in 174 msec 2024-01-01T09:11:20,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=49, resume processing ppid=48 2024-01-01T09:11:20,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=257e5f4c014f577810ee0791199ada41, ASSIGN in 331 msec 2024-01-01T09:11:20,866 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:20,866 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100280866"}]},"ts":"1704100280866"} 2024-01-01T09:11:20,868 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-01-01T09:11:20,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:20,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1760 sec 2024-01-01T09:11:20,947 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433121 expired clientIPAndPort=172.31.12.81:48854, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:11:20,948 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433119 expired clientIPAndPort=172.31.12.81:48854, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:11:20,948 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433118 expired clientIPAndPort=172.31.12.81:48854, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:11:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-01-01T09:11:22,702 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-01-01T09:11:22,702 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-01-01T09:11:22,703 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:22,706 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-01-01T09:11:22,707 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:22,707 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-01-01T09:11:22,709 DEBUG [hconnection-0x1844d13-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:22,710 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59458, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:22,719 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:22,721 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:11:22,722 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:11:22,723 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1ab6ab10 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:11:22,723 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1ab6ab10 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:22,725 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59474, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:22,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:22,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:11:23,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-01-01T09:11:23,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:23,736 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-01-01T09:11:23,736 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-01-01T09:11:23,736 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:11:24,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 1, skipping over this response of 89 bytes 2024-01-01T09:11:24,663 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-01-01T09:11:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:24,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:11:25,725 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms 2024-01-01T09:11:25,726 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms 2024-01-01T09:11:25,726 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:11:25,727 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-01-01T09:11:23.737Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-01-01T09:11:25.726Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=973ms, rpcTimeout=969ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-01-01T09:11:25,729 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3008 2024-01-01T09:11:25,729 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7d8e303d to 127.0.0.1:56255 2024-01-01T09:11:25,730 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:25,730 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:25,730 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:25,730 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:25,730 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ab6ab10 to 127.0.0.1:56255 2024-01-01T09:11:25,742 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=215 (was 217), OpenFileDescriptor=511 (was 514), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=34 (was 37), ProcessCount=171 (was 171), AvailableMemoryMB=5409 (was 5415) 2024-01-01T09:11:25,752 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=215, OpenFileDescriptor=511, MaxFileDescriptor=60000, SystemLoadAverage=34, ProcessCount=171, AvailableMemoryMB=5409 2024-01-01T09:11:25,754 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x508ce181 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:25,758 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x508ce181 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61182199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:25,759 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x79400d05 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:25,763 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6722a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:25,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-01-01T09:11:25,768 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:25,768 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:25,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-01-01T09:11:25,769 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2024-01-01T09:11:26,051 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433112 expired clientIPAndPort=172.31.12.81:48858, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:11:26,182 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 07df45817a4b1a315f53d0436e57ec0e, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:26,253 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:11:26,347 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 07df45817a4b1a315f53d0436e57ec0e, disabling compactions & flushes 2024-01-01T09:11:26,592 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. after waiting 0 ms 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,592 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,592 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 07df45817a4b1a315f53d0436e57ec0e: 2024-01-01T09:11:26,593 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:26,594 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1704100286593"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100286593"}]},"ts":"1704100286593"} 2024-01-01T09:11:26,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:26,596 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:26,596 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100286596"}]},"ts":"1704100286596"} 2024-01-01T09:11:26,598 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:26,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=07df45817a4b1a315f53d0436e57ec0e, ASSIGN}] 2024-01-01T09:11:26,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=07df45817a4b1a315f53d0436e57ec0e, ASSIGN 2024-01-01T09:11:26,611 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=07df45817a4b1a315f53d0436e57ec0e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:26,762 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=07df45817a4b1a315f53d0436e57ec0e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:26,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 07df45817a4b1a315f53d0436e57ec0e, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2024-01-01T09:11:26,915 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:26,920 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 07df45817a4b1a315f53d0436e57ec0e, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:26,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:26,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,922 INFO [StoreOpener-07df45817a4b1a315f53d0436e57ec0e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,924 INFO [StoreOpener-07df45817a4b1a315f53d0436e57ec0e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07df45817a4b1a315f53d0436e57ec0e columnFamilyName testFamily 2024-01-01T09:11:26,924 DEBUG [StoreOpener-07df45817a4b1a315f53d0436e57ec0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:26,924 INFO [StoreOpener-07df45817a4b1a315f53d0436e57ec0e-1 {}] regionserver.HStore(324): Store=07df45817a4b1a315f53d0436e57ec0e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:26,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,926 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,928 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:11:26,930 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:26,931 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 07df45817a4b1a315f53d0436e57ec0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75094607, jitterRate=0.11899684369564056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:26,931 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 07df45817a4b1a315f53d0436e57ec0e: 2024-01-01T09:11:26,932 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2593): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e., pid=53, masterSystemTime=1704100286915 2024-01-01T09:11:26,933 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2620): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,934 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:11:26,934 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=07df45817a4b1a315f53d0436e57ec0e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:26,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=53, resume processing ppid=52 2024-01-01T09:11:26,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 07df45817a4b1a315f53d0436e57ec0e, server=jenkins-hbase3.apache.org,44183,1704100133450 in 172 msec 2024-01-01T09:11:26,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=52, resume processing ppid=51 2024-01-01T09:11:26,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=07df45817a4b1a315f53d0436e57ec0e, ASSIGN in 327 msec 2024-01-01T09:11:26,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:26,940 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100286940"}]},"ts":"1704100286940"} 2024-01-01T09:11:26,942 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:26,944 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:26,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1780 sec 2024-01-01T09:11:28,653 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433107 expired clientIPAndPort=172.31.12.81:42920, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:11:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2024-01-01T09:11:28,773 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-01-01T09:11:28,774 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:28,774 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:28,777 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:28,777 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:28,777 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-01-01T09:11:28,779 DEBUG [hconnection-0xa944c1c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:28,781 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43480, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:28,789 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:28,791 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-01-01T09:11:28,791 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:11:28,793 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x508ce181 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:11:28,793 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x508ce181 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:28,794 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43494, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:28,797 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-01-01T09:11:28,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118558 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,798 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-01-01T09:11:28,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118558 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433058 2024-01-01T09:11:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:43494 deadline: 1704100291797, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:28,799 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118558 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:11:28,800 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-01-01T09:11:28,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118559 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,801 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-01-01T09:11:28,802 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:28,802 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:28,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118560 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433056 2024-01-01T09:11:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:43494 deadline: 1704100291801, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:28,804 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118560 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:11:28,805 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:28,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118561 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6882434854160433055 2024-01-01T09:11:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:43494 deadline: 1704100291804, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:28,806 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118561 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:11:28,806 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:28,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118562 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:28,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6882434854160433054 2024-01-01T09:11:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:43494 deadline: 1704100291806, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:28,809 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118562 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:11:28,810 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:28,810 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x79400d05 to 127.0.0.1:56255 2024-01-01T09:11:28,810 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:28,810 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:28,810 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:28,810 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:28,810 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x508ce181 to 127.0.0.1:56255 2024-01-01T09:11:28,822 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=217 (was 215) - Thread LEAK? -, OpenFileDescriptor=512 (was 511) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=34 (was 34), ProcessCount=171 (was 171), AvailableMemoryMB=5406 (was 5409) 2024-01-01T09:11:28,832 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=217, OpenFileDescriptor=512, MaxFileDescriptor=60000, SystemLoadAverage=34, ProcessCount=171, AvailableMemoryMB=5405 2024-01-01T09:11:28,834 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x04720de6 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:28,838 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x04720de6 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de76a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:28,839 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3185b248 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:28,843 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75243e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:28,845 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:28,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:11:28,849 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:28,849 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:28,850 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:28,850 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-01-01T09:11:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-01-01T09:11:29,265 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 415f19567bc32def48eed0666b8f376c, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 415f19567bc32def48eed0666b8f376c, disabling compactions & flushes 2024-01-01T09:11:29,686 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. after waiting 0 ms 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:29,686 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:29,686 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 415f19567bc32def48eed0666b8f376c: 2024-01-01T09:11:29,687 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:29,687 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1704100289687"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100289687"}]},"ts":"1704100289687"} 2024-01-01T09:11:29,689 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:29,690 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:29,690 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100289690"}]},"ts":"1704100289690"} 2024-01-01T09:11:29,692 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:29,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=415f19567bc32def48eed0666b8f376c, ASSIGN}] 2024-01-01T09:11:29,698 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=415f19567bc32def48eed0666b8f376c, ASSIGN 2024-01-01T09:11:29,699 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=415f19567bc32def48eed0666b8f376c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:29,850 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=415f19567bc32def48eed0666b8f376c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:29,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure 415f19567bc32def48eed0666b8f376c, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-01-01T09:11:30,003 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:30,007 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:30,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7254): Opening region: {ENCODED => 415f19567bc32def48eed0666b8f376c, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:30,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true 415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:30,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7296): checking encryption for 415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7299): checking classloading for 415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,009 INFO [StoreOpener-415f19567bc32def48eed0666b8f376c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,010 INFO [StoreOpener-415f19567bc32def48eed0666b8f376c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 415f19567bc32def48eed0666b8f376c columnFamilyName testFamily 2024-01-01T09:11:30,011 DEBUG [StoreOpener-415f19567bc32def48eed0666b8f376c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:30,011 INFO [StoreOpener-415f19567bc32def48eed0666b8f376c-1 {}] regionserver.HStore(324): Store=415f19567bc32def48eed0666b8f376c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:30,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1076): writing seq id for 415f19567bc32def48eed0666b8f376c 2024-01-01T09:11:30,017 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:30,018 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1093): Opened 415f19567bc32def48eed0666b8f376c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65288247, jitterRate=-0.02712930738925934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:30,018 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(992): Region open journal for 415f19567bc32def48eed0666b8f376c: 2024-01-01T09:11:30,019 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c., pid=56, masterSystemTime=1704100290003 2024-01-01T09:11:30,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:30,021 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:11:30,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=415f19567bc32def48eed0666b8f376c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:30,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=56, resume processing ppid=55 2024-01-01T09:11:30,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure 415f19567bc32def48eed0666b8f376c, server=jenkins-hbase3.apache.org,44183,1704100133450 in 170 msec 2024-01-01T09:11:30,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=55, resume processing ppid=54 2024-01-01T09:11:30,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=415f19567bc32def48eed0666b8f376c, ASSIGN in 327 msec 2024-01-01T09:11:30,027 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:30,027 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100290027"}]},"ts":"1704100290027"} 2024-01-01T09:11:30,029 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:30,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:30,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1860 sec 2024-01-01T09:11:30,655 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433105 expired clientIPAndPort=172.31.12.81:42920, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:11:30,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-01-01T09:11:30,665 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-01-01T09:11:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-01-01T09:11:31,855 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-01-01T09:11:31,855 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:31,855 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:31,859 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:31,859 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:31,859 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-01-01T09:11:31,861 DEBUG [hconnection-0x59b20969-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:31,863 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43510, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:31,870 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:31,871 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:11:31,872 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:11:32,422 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:33,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-01-01T09:11:33,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:33,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:11:33,150 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:36,759 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433100 expired clientIPAndPort=172.31.12.81:42950, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:11:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 6, skipping over this response of 89 bytes 2024-01-01T09:11:38,883 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:11:40,877 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:11:40,878 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-01-01T09:11:40.877Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11009: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1994ms, rpcTimeout=1990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:11:40,878 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-01-01T09:11:40,879 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3185b248 to 127.0.0.1:56255 2024-01-01T09:11:40,879 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:40,879 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:40,879 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:40,879 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:40,880 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x04720de6 to 127.0.0.1:56255 2024-01-01T09:11:40,891 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=213 (was 217), OpenFileDescriptor=509 (was 512), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=67 (was 34) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=5394 (was 5405) 2024-01-01T09:11:40,901 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=213, OpenFileDescriptor=509, MaxFileDescriptor=60000, SystemLoadAverage=67, ProcessCount=171, AvailableMemoryMB=5394 2024-01-01T09:11:40,902 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6db9ac81 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:40,911 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x6db9ac81 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d64774e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:40,912 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x75405e4c to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:40,915 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531f5cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:40,917 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:40,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-01-01T09:11:40,920 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:40,920 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:40,920 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-01-01T09:11:40,920 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2024-01-01T09:11:41,334 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 765fa36cba65fd597c2600619705e027, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 765fa36cba65fd597c2600619705e027, disabling compactions & flushes 2024-01-01T09:11:41,343 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. after waiting 0 ms 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,343 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,343 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 765fa36cba65fd597c2600619705e027: 2024-01-01T09:11:41,344 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:41,344 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1704100301344"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100301344"}]},"ts":"1704100301344"} 2024-01-01T09:11:41,346 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:41,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:41,347 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100301346"}]},"ts":"1704100301346"} 2024-01-01T09:11:41,348 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:41,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=765fa36cba65fd597c2600619705e027, ASSIGN}] 2024-01-01T09:11:41,353 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=765fa36cba65fd597c2600619705e027, ASSIGN 2024-01-01T09:11:41,353 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=765fa36cba65fd597c2600619705e027, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:41,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=765fa36cba65fd597c2600619705e027, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:41,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 765fa36cba65fd597c2600619705e027, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:41,658 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:41,664 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => 765fa36cba65fd597c2600619705e027, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:41,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:41,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for 765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for 765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,666 INFO [StoreOpener-765fa36cba65fd597c2600619705e027-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,668 INFO [StoreOpener-765fa36cba65fd597c2600619705e027-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 765fa36cba65fd597c2600619705e027 columnFamilyName testFamily 2024-01-01T09:11:41,668 DEBUG [StoreOpener-765fa36cba65fd597c2600619705e027-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:41,668 INFO [StoreOpener-765fa36cba65fd597c2600619705e027-1 {}] regionserver.HStore(324): Store=765fa36cba65fd597c2600619705e027/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:41,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,673 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for 765fa36cba65fd597c2600619705e027 2024-01-01T09:11:41,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:41,676 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1093): Opened 765fa36cba65fd597c2600619705e027; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59983126, jitterRate=-0.10618177056312561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:41,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(992): Region open journal for 765fa36cba65fd597c2600619705e027: 2024-01-01T09:11:41,677 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027., pid=59, masterSystemTime=1704100301658 2024-01-01T09:11:41,679 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,679 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:11:41,679 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=765fa36cba65fd597c2600619705e027, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:41,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=59, resume processing ppid=58 2024-01-01T09:11:41,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 765fa36cba65fd597c2600619705e027, server=jenkins-hbase3.apache.org,44183,1704100133450 in 175 msec 2024-01-01T09:11:41,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=58, resume processing ppid=57 2024-01-01T09:11:41,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=765fa36cba65fd597c2600619705e027, ASSIGN in 331 msec 2024-01-01T09:11:41,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:41,686 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100301685"}]},"ts":"1704100301685"} 2024-01-01T09:11:41,687 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:41,691 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:41,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 774 msec 2024-01-01T09:11:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2024-01-01T09:11:41,923 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-01-01T09:11:41,923 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:41,924 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:41,927 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:41,927 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:41,927 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-01-01T09:11:41,930 DEBUG [hconnection-0x53103a33-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:41,932 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44896, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:41,939 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:41,941 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:11:41,941 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:11:41,943 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x6db9ac81 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:11:41,943 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x6db9ac81 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:41,944 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44900, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:11:41,948 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:11:41,948 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118574 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:41,949 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:11:41,949 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118574 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:41,949 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-01-01T09:11:42,666 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-01-01T09:11:43,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-01-01T09:11:43,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:45,383 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:11:47,207 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:50,957 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9007 2024-01-01T09:11:50,957 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x75405e4c to 127.0.0.1:56255 2024-01-01T09:11:50,957 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118574 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:11:50,957 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:50,957 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:50,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433042' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3588) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-01-01T09:11:50,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:44900 deadline: 1704100316957, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433042' does not exist 2024-01-01T09:11:50,957 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:50,957 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:50,958 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6db9ac81 to 127.0.0.1:56255 2024-01-01T09:11:50,959 WARN [RS-EventLoopGroup-3-1 {}] ipc.NettyRpcServerRequestDecoder(47): Connection /172.31.12.81:44900; caught unexpected downstream exception. org.apache.hbase.thirdparty.io.netty.channel.unix.Errors$NativeIoException: recvAddress(..) failed: Connection reset by peer 2024-01-01T09:11:50,958 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,44183,1704100133450 for closing scanner id = -6882434854160433042 for 765fa36cba65fd597c2600619705e027 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:231) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:239) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-01-01T09:11:50,969 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433087 expired clientIPAndPort=172.31.12.81:35082, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:11:50,969 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=215 (was 213) - Thread LEAK? -, OpenFileDescriptor=510 (was 509) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=56 (was 67), ProcessCount=167 (was 171), AvailableMemoryMB=5315 (was 5394) 2024-01-01T09:11:50,979 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=215, OpenFileDescriptor=510, MaxFileDescriptor=60000, SystemLoadAverage=56, ProcessCount=167, AvailableMemoryMB=5315 2024-01-01T09:11:50,980 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x258a302a to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:50,984 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x258a302a {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67672ae9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:50,985 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5fc47a2c to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:50,989 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e17de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:50,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-01-01T09:11:50,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:50,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:50,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-01-01T09:11:50,995 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2024-01-01T09:11:51,408 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f4a3b78bdc31e990f7f520db112b2f63, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:51,450 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:11:51,819 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:51,819 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing f4a3b78bdc31e990f7f520db112b2f63, disabling compactions & flushes 2024-01-01T09:11:51,819 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:51,819 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:51,819 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. after waiting 0 ms 2024-01-01T09:11:51,820 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:51,820 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:51,820 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for f4a3b78bdc31e990f7f520db112b2f63: 2024-01-01T09:11:51,821 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:51,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1704100311821"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100311821"}]},"ts":"1704100311821"} 2024-01-01T09:11:51,823 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:51,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:51,824 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100311823"}]},"ts":"1704100311823"} 2024-01-01T09:11:51,825 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:51,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f4a3b78bdc31e990f7f520db112b2f63, ASSIGN}] 2024-01-01T09:11:51,831 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f4a3b78bdc31e990f7f520db112b2f63, ASSIGN 2024-01-01T09:11:51,832 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f4a3b78bdc31e990f7f520db112b2f63, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:51,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=f4a3b78bdc31e990f7f520db112b2f63, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:51,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure f4a3b78bdc31e990f7f520db112b2f63, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:51,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2024-01-01T09:11:52,137 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:52,143 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(130): Open testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:52,143 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7254): Opening region: {ENCODED => f4a3b78bdc31e990f7f520db112b2f63, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:52,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(888): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:52,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7296): checking encryption for f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7299): checking classloading for f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,145 INFO [StoreOpener-f4a3b78bdc31e990f7f520db112b2f63-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,147 INFO [StoreOpener-f4a3b78bdc31e990f7f520db112b2f63-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4a3b78bdc31e990f7f520db112b2f63 columnFamilyName testFamily 2024-01-01T09:11:52,147 DEBUG [StoreOpener-f4a3b78bdc31e990f7f520db112b2f63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:52,147 INFO [StoreOpener-f4a3b78bdc31e990f7f520db112b2f63-1 {}] regionserver.HStore(324): Store=f4a3b78bdc31e990f7f520db112b2f63/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:52,148 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1076): writing seq id for f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:11:52,154 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:52,154 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1093): Opened f4a3b78bdc31e990f7f520db112b2f63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59329118, jitterRate=-0.11592724919319153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:52,155 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(992): Region open journal for f4a3b78bdc31e990f7f520db112b2f63: 2024-01-01T09:11:52,156 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2593): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63., pid=62, masterSystemTime=1704100312137 2024-01-01T09:11:52,157 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2620): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:52,157 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(158): Opened testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:11:52,158 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=f4a3b78bdc31e990f7f520db112b2f63, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:52,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=62, resume processing ppid=61 2024-01-01T09:11:52,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure f4a3b78bdc31e990f7f520db112b2f63, server=jenkins-hbase3.apache.org,44183,1704100133450 in 174 msec 2024-01-01T09:11:52,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=61, resume processing ppid=60 2024-01-01T09:11:52,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=f4a3b78bdc31e990f7f520db112b2f63, ASSIGN in 332 msec 2024-01-01T09:11:52,163 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:52,163 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100312163"}]},"ts":"1704100312163"} 2024-01-01T09:11:52,164 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:52,166 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:52,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 1.1750 sec 2024-01-01T09:11:53,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-01-01T09:11:53,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:11:53,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2024-01-01T09:11:53,999 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-01-01T09:11:53,999 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:53,999 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:54,003 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:54,003 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:54,003 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-01-01T09:11:54,005 DEBUG [hconnection-0x7f7208d7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:54,007 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:58362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:54,139 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:54,142 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-01-01T09:11:54,144 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-01-01T09:11:54,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118578 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,145 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-01-01T09:11:54,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118578 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433038 2024-01-01T09:11:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:58362 deadline: 1704100317145, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:54,147 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:11:54,149 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-01-01T09:11:54,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118579 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,150 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-01-01T09:11:54,150 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:54,151 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:54,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118580 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-6882434854160433036 2024-01-01T09:11:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:58362 deadline: 1704100317152, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:54,153 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:11:54,155 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:54,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118581 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-6882434854160433035 2024-01-01T09:11:54,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:58362 deadline: 1704100317155, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:54,156 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:11:54,158 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:54,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118582 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:11:54,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-6882434854160433034 2024-01-01T09:11:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:58362 deadline: 1704100317158, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-01-01T09:11:54,159 DEBUG [Listener at localhost/38993 {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:11:54,161 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-01-01T09:11:54,162 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5fc47a2c to 127.0.0.1:56255 2024-01-01T09:11:54,162 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:54,162 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:54,162 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:54,163 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:54,163 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x258a302a to 127.0.0.1:56255 2024-01-01T09:11:54,175 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=217 (was 215) - Thread LEAK? -, OpenFileDescriptor=512 (was 510) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=52 (was 56), ProcessCount=167 (was 167), AvailableMemoryMB=5308 (was 5315) 2024-01-01T09:11:54,185 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=217, OpenFileDescriptor=512, MaxFileDescriptor=60000, SystemLoadAverage=52, ProcessCount=167, AvailableMemoryMB=5308 2024-01-01T09:11:54,186 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x67c6035e to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:54,190 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x67c6035e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234cabe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:54,191 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x162f07cc to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:54,194 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7dd9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:54,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-01-01T09:11:54,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:54,201 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:54,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-01-01T09:11:54,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=63 2024-01-01T09:11:54,212 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2f6c92576d2ad95a8dd660702ba0381a, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:54,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:54,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 2f6c92576d2ad95a8dd660702ba0381a, disabling compactions & flushes 2024-01-01T09:11:54,621 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. after waiting 0 ms 2024-01-01T09:11:54,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,621 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,622 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2f6c92576d2ad95a8dd660702ba0381a: 2024-01-01T09:11:54,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:54,623 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1704100314622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100314622"}]},"ts":"1704100314622"} 2024-01-01T09:11:54,624 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:54,625 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:54,625 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100314625"}]},"ts":"1704100314625"} 2024-01-01T09:11:54,627 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:54,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=2f6c92576d2ad95a8dd660702ba0381a, ASSIGN}] 2024-01-01T09:11:54,633 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=2f6c92576d2ad95a8dd660702ba0381a, ASSIGN 2024-01-01T09:11:54,634 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=2f6c92576d2ad95a8dd660702ba0381a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:54,667 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-01-01T09:11:54,785 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=2f6c92576d2ad95a8dd660702ba0381a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:54,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure 2f6c92576d2ad95a8dd660702ba0381a, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:54,938 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:54,943 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7254): Opening region: {ENCODED => 2f6c92576d2ad95a8dd660702ba0381a, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:54,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:54,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7296): checking encryption for 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7299): checking classloading for 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,945 INFO [StoreOpener-2f6c92576d2ad95a8dd660702ba0381a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,946 INFO [StoreOpener-2f6c92576d2ad95a8dd660702ba0381a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f6c92576d2ad95a8dd660702ba0381a columnFamilyName testFamily 2024-01-01T09:11:54,946 DEBUG [StoreOpener-2f6c92576d2ad95a8dd660702ba0381a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:54,947 INFO [StoreOpener-2f6c92576d2ad95a8dd660702ba0381a-1 {}] regionserver.HStore(324): Store=2f6c92576d2ad95a8dd660702ba0381a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:54,948 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,948 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,950 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1076): writing seq id for 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:11:54,952 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:54,953 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): Opened 2f6c92576d2ad95a8dd660702ba0381a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65386030, jitterRate=-0.025672227144241333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:54,953 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(992): Region open journal for 2f6c92576d2ad95a8dd660702ba0381a: 2024-01-01T09:11:54,954 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a., pid=65, masterSystemTime=1704100314938 2024-01-01T09:11:54,956 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,956 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:11:54,956 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=2f6c92576d2ad95a8dd660702ba0381a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:54,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=65, resume processing ppid=64 2024-01-01T09:11:54,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure 2f6c92576d2ad95a8dd660702ba0381a, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:11:54,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=64, resume processing ppid=63 2024-01-01T09:11:54,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=2f6c92576d2ad95a8dd660702ba0381a, ASSIGN in 328 msec 2024-01-01T09:11:54,962 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:54,963 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100314962"}]},"ts":"1704100314962"} 2024-01-01T09:11:54,964 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:54,966 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:54,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 769 msec 2024-01-01T09:11:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=63 2024-01-01T09:11:55,204 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-01-01T09:11:55,204 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:55,204 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:55,207 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:55,207 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:55,207 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-01-01T09:11:55,209 DEBUG [hconnection-0xa07a764-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:55,211 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36864, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:55,218 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:55,220 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:11:55,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:11:55,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-01-01T09:11:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 7, skipping over this response of 12 bytes 2024-01-01T09:11:57,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:11:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.31.12.81:36864 deadline: 1704100318222, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:11:57,235 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:11:57,236 DEBUG [Listener at localhost/38993 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118587 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:11:57,238 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2017 2024-01-01T09:11:57,238 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x162f07cc to 127.0.0.1:56255 2024-01-01T09:11:57,238 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:57,238 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:11:57,238 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:11:57,238 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:11:57,239 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x67c6035e to 127.0.0.1:56255 2024-01-01T09:11:57,250 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=216 (was 217), OpenFileDescriptor=515 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=52 (was 52), ProcessCount=167 (was 167), AvailableMemoryMB=5305 (was 5308) 2024-01-01T09:11:57,260 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=216, OpenFileDescriptor=515, MaxFileDescriptor=60000, SystemLoadAverage=52, ProcessCount=167, AvailableMemoryMB=5304 2024-01-01T09:11:57,262 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x39875d0e to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:57,267 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x39875d0e {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cabcffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:57,268 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x499b5ec5 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:11:57,271 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9b9cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:11:57,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:11:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:11:57,275 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:11:57,275 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:57,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-01-01T09:11:57,276 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:11:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2024-01-01T09:11:57,645 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:11:57,686 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 48930314afc32fca9384abf242612a24, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 48930314afc32fca9384abf242612a24, disabling compactions & flushes 2024-01-01T09:11:57,695 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. after waiting 0 ms 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:57,695 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:57,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 48930314afc32fca9384abf242612a24: 2024-01-01T09:11:57,696 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:11:57,696 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1704100317696"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100317696"}]},"ts":"1704100317696"} 2024-01-01T09:11:57,698 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:11:57,699 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:11:57,699 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100317699"}]},"ts":"1704100317699"} 2024-01-01T09:11:57,701 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:11:57,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=48930314afc32fca9384abf242612a24, ASSIGN}] 2024-01-01T09:11:57,707 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=48930314afc32fca9384abf242612a24, ASSIGN 2024-01-01T09:11:57,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=48930314afc32fca9384abf242612a24, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:11:57,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=48930314afc32fca9384abf242612a24, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:57,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 48930314afc32fca9384abf242612a24, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:11:58,012 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:58,016 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:58,016 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7254): Opening region: {ENCODED => 48930314afc32fca9384abf242612a24, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:11:58,016 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,016 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:11:58,017 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7296): checking encryption for 48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,017 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7299): checking classloading for 48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,018 INFO [StoreOpener-48930314afc32fca9384abf242612a24-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,020 INFO [StoreOpener-48930314afc32fca9384abf242612a24-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48930314afc32fca9384abf242612a24 columnFamilyName testFamily 2024-01-01T09:11:58,020 DEBUG [StoreOpener-48930314afc32fca9384abf242612a24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:11:58,020 INFO [StoreOpener-48930314afc32fca9384abf242612a24-1 {}] regionserver.HStore(324): Store=48930314afc32fca9384abf242612a24/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:11:58,021 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,021 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,023 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1076): writing seq id for 48930314afc32fca9384abf242612a24 2024-01-01T09:11:58,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:11:58,026 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1093): Opened 48930314afc32fca9384abf242612a24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66817749, jitterRate=-0.0043379515409469604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:11:58,026 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(992): Region open journal for 48930314afc32fca9384abf242612a24: 2024-01-01T09:11:58,027 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24., pid=68, masterSystemTime=1704100318012 2024-01-01T09:11:58,029 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:58,029 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:11:58,029 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=48930314afc32fca9384abf242612a24, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:11:58,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=68, resume processing ppid=67 2024-01-01T09:11:58,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 48930314afc32fca9384abf242612a24, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:11:58,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=67, resume processing ppid=66 2024-01-01T09:11:58,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=48930314afc32fca9384abf242612a24, ASSIGN in 326 msec 2024-01-01T09:11:58,034 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:11:58,034 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100318034"}]},"ts":"1704100318034"} 2024-01-01T09:11:58,035 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:11:58,038 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:11:58,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 765 msec 2024-01-01T09:11:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2024-01-01T09:11:58,278 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-01-01T09:11:58,278 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:11:58,279 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:58,281 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-01-01T09:11:58,282 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:11:58,282 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-01-01T09:11:58,284 DEBUG [hconnection-0x18e84696-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:11:58,285 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36868, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:11:58,292 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:11:58,294 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:11:58,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:11:58,538 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433083 expired clientIPAndPort=172.31.12.81:35086, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:11:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 6, skipping over this response of 89 bytes 2024-01-01T09:12:00,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:12:00,668 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-01-01T09:12:00,668 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-01-01T09:12:01,297 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:12:01,297 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-01-01T09:12:01.297Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:274) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:443) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:317) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:615) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5013: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=991ms, rpcTimeout=990ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:12:01,298 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-01-01T09:12:01,298 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x499b5ec5 to 127.0.0.1:56255 2024-01-01T09:12:01,298 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:01,298 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:01,298 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:01,298 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:01,298 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x39875d0e to 127.0.0.1:56255 2024-01-01T09:12:01,310 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=216 (was 216), OpenFileDescriptor=515 (was 515), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=48 (was 52), ProcessCount=167 (was 167), AvailableMemoryMB=5304 (was 5304) 2024-01-01T09:12:01,320 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=216, OpenFileDescriptor=515, MaxFileDescriptor=60000, SystemLoadAverage=48, ProcessCount=167, AvailableMemoryMB=5304 2024-01-01T09:12:01,321 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x116cd0af to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:01,325 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x116cd0af {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140f4609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:01,326 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0bb9608a to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:01,332 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a1f400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:01,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-01-01T09:12:01,336 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:01,337 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:01,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-01-01T09:12:01,337 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-01-01T09:12:01,748 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 66680db6b4f1ced72c27dcba496ed9e4, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:01,805 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:12:02,159 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:02,159 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 66680db6b4f1ced72c27dcba496ed9e4, disabling compactions & flushes 2024-01-01T09:12:02,159 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,159 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,159 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. after waiting 0 ms 2024-01-01T09:12:02,160 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,160 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,160 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 66680db6b4f1ced72c27dcba496ed9e4: 2024-01-01T09:12:02,161 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:02,161 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1704100322161"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100322161"}]},"ts":"1704100322161"} 2024-01-01T09:12:02,162 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:02,163 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:02,164 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100322163"}]},"ts":"1704100322163"} 2024-01-01T09:12:02,165 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:02,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=66680db6b4f1ced72c27dcba496ed9e4, ASSIGN}] 2024-01-01T09:12:02,170 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=66680db6b4f1ced72c27dcba496ed9e4, ASSIGN 2024-01-01T09:12:02,171 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=66680db6b4f1ced72c27dcba496ed9e4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:02,322 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=66680db6b4f1ced72c27dcba496ed9e4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:02,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure 66680db6b4f1ced72c27dcba496ed9e4, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-01-01T09:12:02,475 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:02,479 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7254): Opening region: {ENCODED => 66680db6b4f1ced72c27dcba496ed9e4, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:02,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:02,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7296): checking encryption for 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7299): checking classloading for 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,482 INFO [StoreOpener-66680db6b4f1ced72c27dcba496ed9e4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,483 INFO [StoreOpener-66680db6b4f1ced72c27dcba496ed9e4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66680db6b4f1ced72c27dcba496ed9e4 columnFamilyName testFamily 2024-01-01T09:12:02,483 DEBUG [StoreOpener-66680db6b4f1ced72c27dcba496ed9e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:02,484 INFO [StoreOpener-66680db6b4f1ced72c27dcba496ed9e4-1 {}] regionserver.HStore(324): Store=66680db6b4f1ced72c27dcba496ed9e4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:02,484 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,485 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1076): writing seq id for 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:12:02,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:02,490 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1093): Opened 66680db6b4f1ced72c27dcba496ed9e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75000912, jitterRate=0.11760067939758301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:02,490 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(992): Region open journal for 66680db6b4f1ced72c27dcba496ed9e4: 2024-01-01T09:12:02,491 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4., pid=71, masterSystemTime=1704100322475 2024-01-01T09:12:02,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,492 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:12:02,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=66680db6b4f1ced72c27dcba496ed9e4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:02,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=71, resume processing ppid=70 2024-01-01T09:12:02,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure 66680db6b4f1ced72c27dcba496ed9e4, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:12:02,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=70, resume processing ppid=69 2024-01-01T09:12:02,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=66680db6b4f1ced72c27dcba496ed9e4, ASSIGN in 327 msec 2024-01-01T09:12:02,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:02,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100322498"}]},"ts":"1704100322498"} 2024-01-01T09:12:02,499 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:02,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:02,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1680 sec 2024-01-01T09:12:03,518 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:03,553 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-01-01T09:12:03,553 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:03,554 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:12:03,554 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:03,554 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-01-01T09:12:03,554 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-01-01T09:12:04,341 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-01-01T09:12:04,341 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:04,342 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:04,345 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:04,345 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:04,345 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-01-01T09:12:04,347 DEBUG [hconnection-0x5f262ab0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:04,348 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36872, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:04,355 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:04,357 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:04,358 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:12:04,359 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x116cd0af {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:12:04,359 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x116cd0af {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:04,361 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36878, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:04,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:12:04,364 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:12:04,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118599 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:04,364 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:12:04,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118599 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:04,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-01-01T09:12:06,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-01-01T09:12:07,366 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3001 2024-01-01T09:12:07,366 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0bb9608a to 127.0.0.1:56255 2024-01-01T09:12:07,366 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118599 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:12:07,366 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:07,367 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:07,367 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:07,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433017' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3588) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-01-01T09:12:07,367 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:07,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.31.12.81:36878 deadline: 1704100328366, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6882434854160433017' does not exist 2024-01-01T09:12:07,367 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x116cd0af to 127.0.0.1:56255 2024-01-01T09:12:07,369 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:12:07,367 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(361): Call to jenkins-hbase3.apache.org,44183,1704100133450 for closing scanner id = -6882434854160433017 for 66680db6b4f1ced72c27dcba496ed9e4 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:229) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:231) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:239) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-01-01T09:12:07,379 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=215 (was 216), OpenFileDescriptor=513 (was 515), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=44 (was 48), ProcessCount=167 (was 167), AvailableMemoryMB=5304 (was 5304) 2024-01-01T09:12:07,389 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=215, OpenFileDescriptor=513, MaxFileDescriptor=60000, SystemLoadAverage=44, ProcessCount=167, AvailableMemoryMB=5304 2024-01-01T09:12:07,390 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7ef83128 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:07,395 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x7ef83128 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab4b200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:07,396 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f00b392 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:07,400 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e05bc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:07,402 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:07,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-01-01T09:12:07,404 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:07,404 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:07,404 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-01-01T09:12:07,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2024-01-01T09:12:07,815 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0295370e633733ae7988ee9d3ebc2908, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:07,865 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:12:08,144 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433077 expired clientIPAndPort=172.31.12.81:46174, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 0295370e633733ae7988ee9d3ebc2908, disabling compactions & flushes 2024-01-01T09:12:08,225 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. after waiting 0 ms 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,225 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0295370e633733ae7988ee9d3ebc2908: 2024-01-01T09:12:08,226 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:08,226 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1704100328226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100328226"}]},"ts":"1704100328226"} 2024-01-01T09:12:08,228 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:08,229 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:08,229 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100328229"}]},"ts":"1704100328229"} 2024-01-01T09:12:08,230 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:08,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0295370e633733ae7988ee9d3ebc2908, ASSIGN}] 2024-01-01T09:12:08,237 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0295370e633733ae7988ee9d3ebc2908, ASSIGN 2024-01-01T09:12:08,237 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0295370e633733ae7988ee9d3ebc2908, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:08,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0295370e633733ae7988ee9d3ebc2908, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:08,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 0295370e633733ae7988ee9d3ebc2908, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2024-01-01T09:12:08,541 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:08,545 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => 0295370e633733ae7988ee9d3ebc2908, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:08,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:08,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,547 INFO [StoreOpener-0295370e633733ae7988ee9d3ebc2908-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,549 INFO [StoreOpener-0295370e633733ae7988ee9d3ebc2908-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0295370e633733ae7988ee9d3ebc2908 columnFamilyName testFamily 2024-01-01T09:12:08,549 DEBUG [StoreOpener-0295370e633733ae7988ee9d3ebc2908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:08,549 INFO [StoreOpener-0295370e633733ae7988ee9d3ebc2908-1 {}] regionserver.HStore(324): Store=0295370e633733ae7988ee9d3ebc2908/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:08,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,551 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,553 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:12:08,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:08,556 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1093): Opened 0295370e633733ae7988ee9d3ebc2908; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66849099, jitterRate=-0.0038708001375198364}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:08,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(992): Region open journal for 0295370e633733ae7988ee9d3ebc2908: 2024-01-01T09:12:08,557 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908., pid=74, masterSystemTime=1704100328541 2024-01-01T09:12:08,558 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,558 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:12:08,559 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=0295370e633733ae7988ee9d3ebc2908, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:08,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=74, resume processing ppid=73 2024-01-01T09:12:08,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 0295370e633733ae7988ee9d3ebc2908, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:12:08,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=73, resume processing ppid=72 2024-01-01T09:12:08,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=0295370e633733ae7988ee9d3ebc2908, ASSIGN in 325 msec 2024-01-01T09:12:08,563 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:08,563 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100328563"}]},"ts":"1704100328563"} 2024-01-01T09:12:08,564 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:08,567 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:08,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 1.1650 sec 2024-01-01T09:12:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2024-01-01T09:12:10,409 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-01-01T09:12:10,409 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:10,410 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:10,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32809 bytes) of info 2024-01-01T09:12:10,416 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118600 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:10,417 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:10,417 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:10,417 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-01-01T09:12:10,419 DEBUG [hconnection-0x32f55a35-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:10,420 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43406, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:10,427 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:10,429 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:10,430 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118603 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:12:10,430 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-01-01T09:12:12,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-01-01T09:12:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-01-01T09:12:13,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:14,083 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:15,149 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433075 expired clientIPAndPort=172.31.12.81:46174, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:12:16,931 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6502 2024-01-01T09:12:16,931 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f00b392 to 127.0.0.1:56255 2024-01-01T09:12:16,931 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:16,931 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:16,931 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:16,931 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:16,932 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7ef83128 to 127.0.0.1:56255 2024-01-01T09:12:16,945 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=215 (was 215), OpenFileDescriptor=510 (was 513), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=37 (was 44), ProcessCount=167 (was 167), AvailableMemoryMB=5304 (was 5304) 2024-01-01T09:12:16,955 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=215, OpenFileDescriptor=510, MaxFileDescriptor=60000, SystemLoadAverage=37, ProcessCount=167, AvailableMemoryMB=5303 2024-01-01T09:12:16,957 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2bb56e54 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:16,961 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x2bb56e54 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f70f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:16,961 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2595c147 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:16,965 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321bef72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:16,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-01-01T09:12:16,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:16,969 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:16,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-01-01T09:12:16,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-01-01T09:12:17,379 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9524465155bd6327e9464a3a91161d37, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 9524465155bd6327e9464a3a91161d37, disabling compactions & flushes 2024-01-01T09:12:17,789 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. after waiting 0 ms 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:17,789 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:17,789 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9524465155bd6327e9464a3a91161d37: 2024-01-01T09:12:17,790 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:17,791 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1704100337791"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100337791"}]},"ts":"1704100337791"} 2024-01-01T09:12:17,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:17,793 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:17,793 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100337793"}]},"ts":"1704100337793"} 2024-01-01T09:12:17,794 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:17,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9524465155bd6327e9464a3a91161d37, ASSIGN}] 2024-01-01T09:12:17,800 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9524465155bd6327e9464a3a91161d37, ASSIGN 2024-01-01T09:12:17,801 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9524465155bd6327e9464a3a91161d37, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:17,952 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=9524465155bd6327e9464a3a91161d37, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:17,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure 9524465155bd6327e9464a3a91161d37, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-01-01T09:12:18,105 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:18,110 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:18,110 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7254): Opening region: {ENCODED => 9524465155bd6327e9464a3a91161d37, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:18,110 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true 9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,110 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:18,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7296): checking encryption for 9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,111 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7299): checking classloading for 9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,112 INFO [StoreOpener-9524465155bd6327e9464a3a91161d37-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,114 INFO [StoreOpener-9524465155bd6327e9464a3a91161d37-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9524465155bd6327e9464a3a91161d37 columnFamilyName testFamily 2024-01-01T09:12:18,114 DEBUG [StoreOpener-9524465155bd6327e9464a3a91161d37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:18,115 INFO [StoreOpener-9524465155bd6327e9464a3a91161d37-1 {}] regionserver.HStore(324): Store=9524465155bd6327e9464a3a91161d37/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:18,116 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,116 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,119 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1076): writing seq id for 9524465155bd6327e9464a3a91161d37 2024-01-01T09:12:18,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:18,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): Opened 9524465155bd6327e9464a3a91161d37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61134460, jitterRate=-0.08902555704116821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:18,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(992): Region open journal for 9524465155bd6327e9464a3a91161d37: 2024-01-01T09:12:18,123 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37., pid=77, masterSystemTime=1704100338105 2024-01-01T09:12:18,124 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:18,124 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:12:18,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=9524465155bd6327e9464a3a91161d37, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:18,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=77, resume processing ppid=76 2024-01-01T09:12:18,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure 9524465155bd6327e9464a3a91161d37, server=jenkins-hbase3.apache.org,44183,1704100133450 in 173 msec 2024-01-01T09:12:18,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=76, resume processing ppid=75 2024-01-01T09:12:18,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=9524465155bd6327e9464a3a91161d37, ASSIGN in 328 msec 2024-01-01T09:12:18,129 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:18,130 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100338130"}]},"ts":"1704100338130"} 2024-01-01T09:12:18,131 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:18,133 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:18,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1660 sec 2024-01-01T09:12:18,671 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-01-01T09:12:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-01-01T09:12:19,973 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-01-01T09:12:19,973 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:19,973 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:19,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32886 bytes) of info 2024-01-01T09:12:19,977 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118604 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:19,978 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:19,979 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:19,979 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-01-01T09:12:19,981 DEBUG [hconnection-0x1c25180b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:19,982 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42542, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:19,989 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:19,990 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:19,992 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118607 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-01-01T09:12:19,992 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-01-01T09:12:20,153 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433070 expired clientIPAndPort=172.31.12.81:35198, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:12:21,492 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-01-01T09:12:21,493 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2595c147 to 127.0.0.1:56255 2024-01-01T09:12:21,493 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:21,493 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:21,493 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:21,493 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:21,493 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2bb56e54 to 127.0.0.1:56255 2024-01-01T09:12:21,505 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=218 (was 215) - Thread LEAK? -, OpenFileDescriptor=512 (was 510) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=34 (was 37), ProcessCount=167 (was 167), AvailableMemoryMB=5302 (was 5303) 2024-01-01T09:12:21,515 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=218, OpenFileDescriptor=512, MaxFileDescriptor=60000, SystemLoadAverage=34, ProcessCount=167, AvailableMemoryMB=5302 2024-01-01T09:12:21,516 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3e3c9916 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:21,520 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x3e3c9916 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447d25f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:21,521 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x42588e00 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:21,526 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca357d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:21,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-01-01T09:12:21,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:21,530 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:21,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-01-01T09:12:21,531 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2024-01-01T09:12:21,942 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9033d95dea4b7b8bd8a3d8f91abb3f89, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 9033d95dea4b7b8bd8a3d8f91abb3f89, disabling compactions & flushes 2024-01-01T09:12:22,352 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. after waiting 0 ms 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,352 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,352 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9033d95dea4b7b8bd8a3d8f91abb3f89: 2024-01-01T09:12:22,353 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:22,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1704100342353"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100342353"}]},"ts":"1704100342353"} 2024-01-01T09:12:22,355 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:22,356 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:22,356 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100342356"}]},"ts":"1704100342356"} 2024-01-01T09:12:22,357 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:22,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9033d95dea4b7b8bd8a3d8f91abb3f89, ASSIGN}] 2024-01-01T09:12:22,362 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9033d95dea4b7b8bd8a3d8f91abb3f89, ASSIGN 2024-01-01T09:12:22,362 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9033d95dea4b7b8bd8a3d8f91abb3f89, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:22,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=9033d95dea4b7b8bd8a3d8f91abb3f89, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:22,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 9033d95dea4b7b8bd8a3d8f91abb3f89, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2024-01-01T09:12:22,666 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:22,670 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,670 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7254): Opening region: {ENCODED => 9033d95dea4b7b8bd8a3d8f91abb3f89, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:22,671 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,671 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:22,671 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7296): checking encryption for 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,671 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7299): checking classloading for 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,673 INFO [StoreOpener-9033d95dea4b7b8bd8a3d8f91abb3f89-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,675 INFO [StoreOpener-9033d95dea4b7b8bd8a3d8f91abb3f89-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9033d95dea4b7b8bd8a3d8f91abb3f89 columnFamilyName testFamily 2024-01-01T09:12:22,675 DEBUG [StoreOpener-9033d95dea4b7b8bd8a3d8f91abb3f89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:22,675 INFO [StoreOpener-9033d95dea4b7b8bd8a3d8f91abb3f89-1 {}] regionserver.HStore(324): Store=9033d95dea4b7b8bd8a3d8f91abb3f89/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:22,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,678 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1076): writing seq id for 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:12:22,680 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:22,681 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): Opened 9033d95dea4b7b8bd8a3d8f91abb3f89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61364416, jitterRate=-0.08559894561767578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:22,681 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(992): Region open journal for 9033d95dea4b7b8bd8a3d8f91abb3f89: 2024-01-01T09:12:22,682 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89., pid=80, masterSystemTime=1704100342666 2024-01-01T09:12:22,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,684 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:12:22,684 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=9033d95dea4b7b8bd8a3d8f91abb3f89, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:22,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=80, resume processing ppid=79 2024-01-01T09:12:22,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 9033d95dea4b7b8bd8a3d8f91abb3f89, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:12:22,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=79, resume processing ppid=78 2024-01-01T09:12:22,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=9033d95dea4b7b8bd8a3d8f91abb3f89, ASSIGN in 326 msec 2024-01-01T09:12:22,689 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:22,689 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100342689"}]},"ts":"1704100342689"} 2024-01-01T09:12:22,690 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:22,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:22,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1700 sec 2024-01-01T09:12:23,612 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:23,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-01-01T09:12:23,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:23,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-01-01T09:12:23,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:24,256 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433065 expired clientIPAndPort=172.31.12.81:59474, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:12:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2024-01-01T09:12:24,534 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-01-01T09:12:24,534 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:24,534 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:24,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32859 bytes) of info 2024-01-01T09:12:24,538 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118608 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:24,539 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:24,539 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:24,539 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-01-01T09:12:24,541 DEBUG [hconnection-0x48a98571-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:24,543 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42552, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:24,550 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:24,552 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:24,553 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:12:24,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118611 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:24,555 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:12:24,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118611 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:24,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-01-01T09:12:24,672 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-01-01T09:12:26,258 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433063 expired clientIPAndPort=172.31.12.81:59474, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:12:27,567 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4014: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4014: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3010ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:12:27,568 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3013 2024-01-01T09:12:27,568 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x42588e00 to 127.0.0.1:56255 2024-01-01T09:12:27,568 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:27,568 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:27,568 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:27,568 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:27,569 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3e3c9916 to 127.0.0.1:56255 2024-01-01T09:12:27,580 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=215 (was 218), OpenFileDescriptor=513 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=32 (was 34), ProcessCount=167 (was 167), AvailableMemoryMB=5300 (was 5302) 2024-01-01T09:12:27,590 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=215, OpenFileDescriptor=513, MaxFileDescriptor=60000, SystemLoadAverage=32, ProcessCount=167, AvailableMemoryMB=5300 2024-01-01T09:12:27,591 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x085a6ad3 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:27,594 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x085a6ad3 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e45f340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:27,595 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x76ab81ab to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:27,599 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8896a8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:27,602 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:27,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-01-01T09:12:27,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:27,604 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:27,604 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-01-01T09:12:27,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-01-01T09:12:28,017 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2c9944b1c50d845091a1dba2725ec2ba, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:28,056 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 2c9944b1c50d845091a1dba2725ec2ba, disabling compactions & flushes 2024-01-01T09:12:28,426 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. after waiting 0 ms 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,426 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,426 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2c9944b1c50d845091a1dba2725ec2ba: 2024-01-01T09:12:28,427 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:28,428 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1704100348427"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100348427"}]},"ts":"1704100348427"} 2024-01-01T09:12:28,429 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:28,430 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:28,430 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100348430"}]},"ts":"1704100348430"} 2024-01-01T09:12:28,431 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:28,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=2c9944b1c50d845091a1dba2725ec2ba, ASSIGN}] 2024-01-01T09:12:28,436 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=2c9944b1c50d845091a1dba2725ec2ba, ASSIGN 2024-01-01T09:12:28,437 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=2c9944b1c50d845091a1dba2725ec2ba, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:28,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=2c9944b1c50d845091a1dba2725ec2ba, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:28,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 2c9944b1c50d845091a1dba2725ec2ba, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-01-01T09:12:28,741 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:28,745 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 2c9944b1c50d845091a1dba2725ec2ba, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:28,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:28,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,748 INFO [StoreOpener-2c9944b1c50d845091a1dba2725ec2ba-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,749 INFO [StoreOpener-2c9944b1c50d845091a1dba2725ec2ba-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9944b1c50d845091a1dba2725ec2ba columnFamilyName testFamily 2024-01-01T09:12:28,749 DEBUG [StoreOpener-2c9944b1c50d845091a1dba2725ec2ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:28,750 INFO [StoreOpener-2c9944b1c50d845091a1dba2725ec2ba-1 {}] regionserver.HStore(324): Store=2c9944b1c50d845091a1dba2725ec2ba/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:28,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:12:28,755 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:28,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1093): Opened 2c9944b1c50d845091a1dba2725ec2ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66751402, jitterRate=-0.005326598882675171}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:28,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 2c9944b1c50d845091a1dba2725ec2ba: 2024-01-01T09:12:28,757 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba., pid=83, masterSystemTime=1704100348740 2024-01-01T09:12:28,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,759 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:12:28,759 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=2c9944b1c50d845091a1dba2725ec2ba, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:28,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=83, resume processing ppid=82 2024-01-01T09:12:28,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 2c9944b1c50d845091a1dba2725ec2ba, server=jenkins-hbase3.apache.org,44183,1704100133450 in 172 msec 2024-01-01T09:12:28,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=82, resume processing ppid=81 2024-01-01T09:12:28,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=2c9944b1c50d845091a1dba2725ec2ba, ASSIGN in 327 msec 2024-01-01T09:12:28,764 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:28,764 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100348764"}]},"ts":"1704100348764"} 2024-01-01T09:12:28,765 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:28,768 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:28,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1660 sec 2024-01-01T09:12:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-01-01T09:12:30,608 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-01-01T09:12:30,608 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:30,608 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:30,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2024-01-01T09:12:30,612 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118612 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:30,614 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:30,614 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:30,614 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-01-01T09:12:30,616 DEBUG [hconnection-0x5acf857-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:30,617 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47140, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:30,624 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:30,626 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:30,628 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:12:30,628 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-01-01T09:12:30,672 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-01-01T09:12:33,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-01-01T09:12:33,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:34,285 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:37,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 7, skipping over this response of 12 bytes 2024-01-01T09:12:37,646 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:12:37,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.31.12.81:47140 deadline: 1704100359627, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-01-01T09:12:37,647 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-01-01T09:12:37,648 DEBUG [Listener at localhost/38993 {}] client.ClientScanner(599): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor63.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 11564309219549118615 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3213) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3596) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45006) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2024-01-01T09:12:37,649 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7022 2024-01-01T09:12:37,649 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x76ab81ab to 127.0.0.1:56255 2024-01-01T09:12:37,650 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:37,650 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:37,650 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:37,650 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:37,650 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x085a6ad3 to 127.0.0.1:56255 2024-01-01T09:12:37,661 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=215 (was 215), OpenFileDescriptor=510 (was 513), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=27 (was 32), ProcessCount=169 (was 167) - ProcessCount LEAK? -, AvailableMemoryMB=5293 (was 5300) 2024-01-01T09:12:37,671 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=215, OpenFileDescriptor=510, MaxFileDescriptor=60000, SystemLoadAverage=27, ProcessCount=169, AvailableMemoryMB=5293 2024-01-01T09:12:37,672 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x79deccf4 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:37,677 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x79deccf4 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6700dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:37,678 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63473c7f to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:37,682 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50239911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:37,684 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:37,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-01-01T09:12:37,686 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:37,686 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:37,686 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-01-01T09:12:37,686 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=84 2024-01-01T09:12:38,097 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 44e29dca8b8d85e87b669d8fa8e89f6f, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:38,467 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433049 expired clientIPAndPort=172.31.12.81:43510, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:12:38,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:38,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 44e29dca8b8d85e87b669d8fa8e89f6f, disabling compactions & flushes 2024-01-01T09:12:38,506 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. after waiting 0 ms 2024-01-01T09:12:38,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,507 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 44e29dca8b8d85e87b669d8fa8e89f6f: 2024-01-01T09:12:38,508 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:38,508 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1704100358508"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100358508"}]},"ts":"1704100358508"} 2024-01-01T09:12:38,509 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:38,510 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:38,510 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100358510"}]},"ts":"1704100358510"} 2024-01-01T09:12:38,511 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:38,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=44e29dca8b8d85e87b669d8fa8e89f6f, ASSIGN}] 2024-01-01T09:12:38,516 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=44e29dca8b8d85e87b669d8fa8e89f6f, ASSIGN 2024-01-01T09:12:38,516 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=44e29dca8b8d85e87b669d8fa8e89f6f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:38,667 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=44e29dca8b8d85e87b669d8fa8e89f6f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:38,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure 44e29dca8b8d85e87b669d8fa8e89f6f, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=84 2024-01-01T09:12:38,820 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:38,827 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7254): Opening region: {ENCODED => 44e29dca8b8d85e87b669d8fa8e89f6f, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:38,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:38,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7296): checking encryption for 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7299): checking classloading for 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,829 INFO [StoreOpener-44e29dca8b8d85e87b669d8fa8e89f6f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,831 INFO [StoreOpener-44e29dca8b8d85e87b669d8fa8e89f6f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44e29dca8b8d85e87b669d8fa8e89f6f columnFamilyName testFamily 2024-01-01T09:12:38,831 DEBUG [StoreOpener-44e29dca8b8d85e87b669d8fa8e89f6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:38,831 INFO [StoreOpener-44e29dca8b8d85e87b669d8fa8e89f6f-1 {}] regionserver.HStore(324): Store=44e29dca8b8d85e87b669d8fa8e89f6f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:38,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,835 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1076): writing seq id for 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:12:38,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:38,837 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1093): Opened 44e29dca8b8d85e87b669d8fa8e89f6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71791228, jitterRate=0.06977266073226929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:38,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(992): Region open journal for 44e29dca8b8d85e87b669d8fa8e89f6f: 2024-01-01T09:12:38,838 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., pid=86, masterSystemTime=1704100358820 2024-01-01T09:12:38,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,840 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:12:38,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=44e29dca8b8d85e87b669d8fa8e89f6f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:38,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=86, resume processing ppid=85 2024-01-01T09:12:38,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure 44e29dca8b8d85e87b669d8fa8e89f6f, server=jenkins-hbase3.apache.org,44183,1704100133450 in 173 msec 2024-01-01T09:12:38,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=85, resume processing ppid=84 2024-01-01T09:12:38,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=44e29dca8b8d85e87b669d8fa8e89f6f, ASSIGN in 329 msec 2024-01-01T09:12:38,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:38,846 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100358846"}]},"ts":"1704100358846"} 2024-01-01T09:12:38,848 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:38,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:38,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1670 sec 2024-01-01T09:12:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=84 2024-01-01T09:12:40,690 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-01-01T09:12:40,690 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:40,690 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:40,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32913 bytes) of info 2024-01-01T09:12:40,693 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118617 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:40,694 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:40,695 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:40,695 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-01-01T09:12:40,696 DEBUG [hconnection-0x4d42a879-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:40,698 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38888, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:40,705 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:40,707 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:12:40,707 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:12:40,708 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x79deccf4 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:12:40,709 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x79deccf4 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:40,710 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38892, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:40,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:12:40,712 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:12:42,674 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-01-01T09:12:43,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-01-01T09:12:43,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:44,329 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:45,472 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433047 expired clientIPAndPort=172.31.12.81:43510, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:12:46,715 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-01-01T09:12:46,715 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-01-01T09:12:46,716 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:12:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 1, skipping over this response of 89 bytes 2024-01-01T09:12:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:12:47,734 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-01-01T09:12:49,715 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1982ms, rpcTimeout=1974ms 2024-01-01T09:12:49,716 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1982ms, rpcTimeout=1974ms 2024-01-01T09:12:49,716 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:12:49,716 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-01-01T09:12:46.716Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6004ms, rpcTimeout=6000ms 2024-01-01T09:12:49.716Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1982ms, rpcTimeout=1974ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1982ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1982ms, rpcTimeout=1974ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-01-01T09:12:49,719 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2024-01-01T09:12:49,719 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x63473c7f to 127.0.0.1:56255 2024-01-01T09:12:49,719 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:49,719 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:12:49,719 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:12:49,719 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:12:49,719 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x79deccf4 to 127.0.0.1:56255 2024-01-01T09:12:49,731 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=213 (was 215), OpenFileDescriptor=506 (was 510), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=21 (was 27), ProcessCount=167 (was 169), AvailableMemoryMB=5298 (was 5293) - AvailableMemoryMB LEAK? - 2024-01-01T09:12:49,741 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=213, OpenFileDescriptor=506, MaxFileDescriptor=60000, SystemLoadAverage=21, ProcessCount=167, AvailableMemoryMB=5298 2024-01-01T09:12:49,742 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5285f209 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:49,745 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x5285f209 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c0b915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:49,746 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x428f8784 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:12:49,750 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b800ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:12:49,752 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:12:49,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-01-01T09:12:49,754 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:12:49,754 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:49,754 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-01-01T09:12:49,755 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:12:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-01-01T09:12:50,166 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fd7437d2995aea437ed3772b0263c069, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing fd7437d2995aea437ed3772b0263c069, disabling compactions & flushes 2024-01-01T09:12:50,577 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. after waiting 0 ms 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,577 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,577 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for fd7437d2995aea437ed3772b0263c069: 2024-01-01T09:12:50,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:12:50,578 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1704100370578"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100370578"}]},"ts":"1704100370578"} 2024-01-01T09:12:50,580 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:12:50,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:12:50,580 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100370580"}]},"ts":"1704100370580"} 2024-01-01T09:12:50,582 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:12:50,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=fd7437d2995aea437ed3772b0263c069, ASSIGN}] 2024-01-01T09:12:50,586 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=fd7437d2995aea437ed3772b0263c069, ASSIGN 2024-01-01T09:12:50,587 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=fd7437d2995aea437ed3772b0263c069, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:12:50,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=fd7437d2995aea437ed3772b0263c069, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:50,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure fd7437d2995aea437ed3772b0263c069, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:12:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-01-01T09:12:50,891 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:50,896 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(130): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7254): Opening region: {ENCODED => fd7437d2995aea437ed3772b0263c069, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:12:50,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(888): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:12:50,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7296): checking encryption for fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,897 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7299): checking classloading for fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,898 INFO [StoreOpener-fd7437d2995aea437ed3772b0263c069-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,900 INFO [StoreOpener-fd7437d2995aea437ed3772b0263c069-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd7437d2995aea437ed3772b0263c069 columnFamilyName testFamily 2024-01-01T09:12:50,900 DEBUG [StoreOpener-fd7437d2995aea437ed3772b0263c069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:12:50,900 INFO [StoreOpener-fd7437d2995aea437ed3772b0263c069-1 {}] regionserver.HStore(324): Store=fd7437d2995aea437ed3772b0263c069/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:12:50,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1076): writing seq id for fd7437d2995aea437ed3772b0263c069 2024-01-01T09:12:50,905 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:12:50,906 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1093): Opened fd7437d2995aea437ed3772b0263c069; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60790097, jitterRate=-0.09415696561336517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:12:50,906 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(992): Region open journal for fd7437d2995aea437ed3772b0263c069: 2024-01-01T09:12:50,907 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069., pid=89, masterSystemTime=1704100370891 2024-01-01T09:12:50,908 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,908 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(158): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:12:50,909 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=fd7437d2995aea437ed3772b0263c069, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:12:50,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=89, resume processing ppid=88 2024-01-01T09:12:50,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1461): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure fd7437d2995aea437ed3772b0263c069, server=jenkins-hbase3.apache.org,44183,1704100133450 in 171 msec 2024-01-01T09:12:50,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=88, resume processing ppid=87 2024-01-01T09:12:50,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1461): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=fd7437d2995aea437ed3772b0263c069, ASSIGN in 327 msec 2024-01-01T09:12:50,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:12:50,914 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100370914"}]},"ts":"1704100370914"} 2024-01-01T09:12:50,915 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:12:50,918 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:12:50,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1660 sec 2024-01-01T09:12:51,476 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433042 expired clientIPAndPort=172.31.12.81:44900, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:12:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-01-01T09:12:52,758 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-01-01T09:12:52,758 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:12:52,758 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:52,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32861 bytes) of info 2024-01-01T09:12:52,762 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118624 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:52,763 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-01-01T09:12:52,763 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:12:52,763 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-01-01T09:12:52,765 DEBUG [hconnection-0x2914deea-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:12:52,766 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34790, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:12:52,774 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:12:52,775 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-01-01T09:12:52,777 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-01-01T09:12:52,778 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118627 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:52,779 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-01-01T09:12:52,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118627 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:12:52,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-01-01T09:12:53,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-01-01T09:12:53,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:12:54,178 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433036 expired clientIPAndPort=172.31.12.81:58362, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:12:54,178 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433035 expired clientIPAndPort=172.31.12.81:58362, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:12:54,178 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433038 expired clientIPAndPort=172.31.12.81:58362, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:12:54,234 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:12:54,675 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-01-01T09:12:56,439 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:12:57,281 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433029 expired clientIPAndPort=172.31.12.81:36864, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:12:59,883 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433024 expired clientIPAndPort=172.31.12.81:36868, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:01,788 DEBUG [Listener at localhost/38993 {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10011: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9004ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ... 6 more 2024-01-01T09:13:01,788 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2024-01-01T09:13:01,788 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x428f8784 to 127.0.0.1:56255 2024-01-01T09:13:01,789 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:01,789 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:13:01,789 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:13:01,789 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:01,789 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5285f209 to 127.0.0.1:56255 2024-01-01T09:13:01,801 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=214 (was 213) - Thread LEAK? -, OpenFileDescriptor=508 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=17 (was 21), ProcessCount=167 (was 167), AvailableMemoryMB=5299 (was 5298) - AvailableMemoryMB LEAK? - 2024-01-01T09:13:01,810 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=214, OpenFileDescriptor=508, MaxFileDescriptor=60000, SystemLoadAverage=17, ProcessCount=167, AvailableMemoryMB=5298 2024-01-01T09:13:01,811 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f52c712 to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:13:01,815 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1f52c712 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f237b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:13:01,816 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x48d4a75f to 127.0.0.1:56255 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-01-01T09:13:01,819 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785b57ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-01-01T09:13:01,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.HMaster$4(2339): Client=jenkins//172.31.12.81 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-01-01T09:13:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] procedure2.ProcedureExecutor(1079): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-01-01T09:13:01,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-01-01T09:13:01,823 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:13:01,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-01-01T09:13:01,824 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-01-01T09:13:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=90 2024-01-01T09:13:01,835 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 44e98b595fab849d235acbb2941a22f8, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1668): Closing 44e98b595fab849d235acbb2941a22f8, disabling compactions & flushes 2024-01-01T09:13:01,844 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. after waiting 0 ms 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:01,844 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:01,844 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1622): Region close journal for 44e98b595fab849d235acbb2941a22f8: 2024-01-01T09:13:01,845 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-01-01T09:13:01,845 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1704100381845"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1704100381845"}]},"ts":"1704100381845"} 2024-01-01T09:13:01,847 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-01-01T09:13:01,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-01-01T09:13:01,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100381848"}]},"ts":"1704100381848"} 2024-01-01T09:13:01,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-01-01T09:13:01,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=44e98b595fab849d235acbb2941a22f8, ASSIGN}] 2024-01-01T09:13:01,855 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=44e98b595fab849d235acbb2941a22f8, ASSIGN 2024-01-01T09:13:01,856 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=44e98b595fab849d235acbb2941a22f8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44183,1704100133450; forceNewPlan=false, retain=false 2024-01-01T09:13:01,884 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433022 expired clientIPAndPort=172.31.12.81:36868, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:02,006 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=44e98b595fab849d235acbb2941a22f8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:13:02,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1800): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 44e98b595fab849d235acbb2941a22f8, server=jenkins-hbase3.apache.org,44183,1704100133450}] 2024-01-01T09:13:02,159 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(715): New admin connection to jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:13:02,163 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(130): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:02,163 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7254): Opening region: {ENCODED => 44e98b595fab849d235acbb2941a22f8, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8.', STARTKEY => '', ENDKEY => ''} 2024-01-01T09:13:02,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(888): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-01-01T09:13:02,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7296): checking encryption for 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7299): checking classloading for 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,165 INFO [StoreOpener-44e98b595fab849d235acbb2941a22f8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,167 INFO [StoreOpener-44e98b595fab849d235acbb2941a22f8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44e98b595fab849d235acbb2941a22f8 columnFamilyName testFamily 2024-01-01T09:13:02,167 DEBUG [StoreOpener-44e98b595fab849d235acbb2941a22f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-01-01T09:13:02,167 INFO [StoreOpener-44e98b595fab849d235acbb2941a22f8-1 {}] regionserver.HStore(324): Store=44e98b595fab849d235acbb2941a22f8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-01-01T09:13:02,168 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,169 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1076): writing seq id for 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:02,173 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-01-01T09:13:02,173 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): Opened 44e98b595fab849d235acbb2941a22f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64602492, jitterRate=-0.03734785318374634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-01-01T09:13:02,174 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(992): Region open journal for 44e98b595fab849d235acbb2941a22f8: 2024-01-01T09:13:02,174 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2593): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., pid=92, masterSystemTime=1704100382159 2024-01-01T09:13:02,176 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2620): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:02,176 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(158): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:02,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=44e98b595fab849d235acbb2941a22f8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:13:02,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=92, resume processing ppid=91 2024-01-01T09:13:02,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1461): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 44e98b595fab849d235acbb2941a22f8, server=jenkins-hbase3.apache.org,44183,1704100133450 in 169 msec 2024-01-01T09:13:02,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1957): Finished subprocedure pid=91, resume processing ppid=90 2024-01-01T09:13:02,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1461): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=44e98b595fab849d235acbb2941a22f8, ASSIGN in 324 msec 2024-01-01T09:13:02,180 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-01-01T09:13:02,181 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1704100382180"}]},"ts":"1704100382180"} 2024-01-01T09:13:02,182 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-01-01T09:13:02,185 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-01-01T09:13:02,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1461): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 364 msec 2024-01-01T09:13:02,281 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:13:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46807 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=90 2024-01-01T09:13:02,827 INFO [Listener at localhost/38993 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-01-01T09:13:02,827 DEBUG [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-01-01T09:13:02,827 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:13:02,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44183 {}] regionserver.StoreScanner(1114): Switch to stream read (scanned=32921 bytes) of info 2024-01-01T09:13:02,830 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 11564309219549118628 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-01-01T09:13:02,832 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3581): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-01-01T09:13:02,832 INFO [Listener at localhost/38993 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-01-01T09:13:02,832 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(3601): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-01-01T09:13:02,834 DEBUG [hconnection-0x6a55b7f3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:13:02,835 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42698, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:13:02,843 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-01-01T09:13:02,844 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-01-01T09:13:02,845 DEBUG [Listener at localhost/38993 {}] client.ConnectionUtils(572): Start fetching meta region location from registry 2024-01-01T09:13:02,846 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1f52c712 {}] client.ConnectionUtils(580): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=-1] 2024-01-01T09:13:02,846 DEBUG [ReadOnlyZKClient-127.0.0.1:56255@0x1f52c712 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2024-01-01T09:13:02,848 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42704, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-01-01T09:13:02,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:13:02,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:13:03,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-01-01T09:13:03,149 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-01-01T09:13:03,856 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-01-01T09:13:03,856 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-01-01T09:13:03,856 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:13:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.NettyRpcDuplexHandler(192): Unknown callId: 1, skipping over this response of 89 bytes 2024-01-01T09:13:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2] 2024-01-01T09:13:04,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44183 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-01-01T09:13:05,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms 2024-01-01T09:13:05,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms 2024-01-01T09:13:05,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., hostname=jenkins-hbase3.apache.org,44183,1704100133450, seqNum=2 from cache 2024-01-01T09:13:05,847 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-01-01T09:13:03.856Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-01-01T09:13:05.846Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at java.lang.Thread.getStackTrace(Thread.java:1564) ~[?:1.8.0_362] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:153) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) ~[?:1.8.0_362] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) ~[?:1.8.0_362] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:79) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:70) ~[hbase-shaded-protobuf-4.1.5.jar:4.1.5] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:396) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:139) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=jenkins-hbase3.apache.org:44183 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:233) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=974ms, rpcTimeout=972ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-01-01T09:13:05,849 INFO [Listener at localhost/38993 {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3004 2024-01-01T09:13:05,849 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x48d4a75f to 127.0.0.1:56255 2024-01-01T09:13:05,849 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:05,849 INFO [Listener at localhost/38993 {}] client.AsyncConnectionImpl(225): Connection has been closed by Listener at localhost/38993. 2024-01-01T09:13:05,849 DEBUG [Listener at localhost/38993 {}] client.AsyncConnectionImpl(253): Call stack: at java.lang.Thread.getStackTrace(Thread.java:1564) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at sun.reflect.GeneratedMethodAccessor123.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.lang.Thread.run(Thread.java:750) 2024-01-01T09:13:05,849 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:05,849 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f52c712 to 127.0.0.1:56255 2024-01-01T09:13:05,861 INFO [Listener at localhost/38993 {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=215 (was 214) - Thread LEAK? -, OpenFileDescriptor=510 (was 508) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=16 (was 17), ProcessCount=167 (was 167), AvailableMemoryMB=5298 (was 5298) 2024-01-01T09:13:05,861 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-01-01T09:13:05,861 INFO [Listener at localhost/38993 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2024-01-01T09:13:05,862 DEBUG [Listener at localhost/38993 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2655594c to 127.0.0.1:56255 2024-01-01T09:13:05,862 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:05,862 DEBUG [Listener at localhost/38993 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-01-01T09:13:05,862 DEBUG [Listener at localhost/38993 {}] util.JVMClusterUtil(257): Found active master hash=1351708317, stopped=false 2024-01-01T09:13:05,862 INFO [Listener at localhost/38993 {}] master.ServerManager(901): Cluster shutdown requested of master=jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:13:05,864 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-01-01T09:13:05,864 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-01-01T09:13:05,864 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:13:05,864 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:13:05,864 INFO [Listener at localhost/38993 {}] procedure2.ProcedureExecutor(677): Stopping 2024-01-01T09:13:05,864 DEBUG [Listener at localhost/38993 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:05,864 INFO [Listener at localhost/38993 {}] regionserver.HRegionServer(2554): ***** STOPPING region server 'jenkins-hbase3.apache.org,44183,1704100133450' ***** 2024-01-01T09:13:05,865 INFO [Listener at localhost/38993 {}] regionserver.HRegionServer(2568): STOPPED: Shutdown requested 2024-01-01T09:13:05,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-01-01T09:13:05,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-01-01T09:13:05,865 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HeapMemoryManager(220): Stopping 2024-01-01T09:13:05,865 INFO [RS:0;jenkins-hbase3:44183 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-01-01T09:13:05,865 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-01-01T09:13:05,865 INFO [RS:0;jenkins-hbase3:44183 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-01-01T09:13:05,865 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 9033d95dea4b7b8bd8a3d8f91abb3f89 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 07df45817a4b1a315f53d0436e57ec0e 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for c15775553a47462f4ada40853ab24526 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 48930314afc32fca9384abf242612a24 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 0295370e633733ae7988ee9d3ebc2908 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 415f19567bc32def48eed0666b8f376c 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 9a4fa884b1f0e010faf65e75081899ff 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 765fa36cba65fd597c2600619705e027 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 2f6c92576d2ad95a8dd660702ba0381a 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 1db8a2d69b3d0f098a78503fcb88cb43 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 1c9c1fffb4d91d95f7adedde98457b0a 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 66680db6b4f1ced72c27dcba496ed9e4 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 84af1702ea2d10a3f3366f7bcc7c8e30 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 44e29dca8b8d85e87b669d8fa8e89f6f 2024-01-01T09:13:05,866 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for f4a3b78bdc31e990f7f520db112b2f63 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 9524465155bd6327e9464a3a91161d37 2024-01-01T09:13:05,867 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9033d95dea4b7b8bd8a3d8f91abb3f89, disabling compactions & flushes 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 5c3468183b42b1b15d589cef672ad7b1 2024-01-01T09:13:05,867 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 2cb844b5df2310f6a3c3ae750e9d60fb 2024-01-01T09:13:05,867 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 2c9944b1c50d845091a1dba2725ec2ba 2024-01-01T09:13:05,867 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. after waiting 0 ms 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:05,867 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for b12eec449640ec3fe233eaeeed122249 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 029d4c0e35227b01f80bf9b14d52df3a 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 44e98b595fab849d235acbb2941a22f8 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 8b769a5133bcc30546aabb30cc7e069f 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 345536859393f3bdc499b7c4b1b8bb53 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 8fe5994cdb848b90b5ea2670beebb22f 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 257e5f4c014f577810ee0791199ada41 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1220): stopping server jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:13:05,867 DEBUG [RS:0;jenkins-hbase3:44183 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(3565): Received CLOSE for 1588230740 2024-01-01T09:13:05,867 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9033d95dea4b7b8bd8a3d8f91abb3f89 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:05,867 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 30 regions to close 2024-01-01T09:13:05,868 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2024-01-01T09:13:05,868 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2024-01-01T09:13:05,868 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-01-01T09:13:05,868 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-01-01T09:13:05,868 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2024-01-01T09:13:05,868 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=48.69 KB heapSize=75.08 KB 2024-01-01T09:13:05,869 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 07df45817a4b1a315f53d0436e57ec0e, 1588230740, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9033d95dea4b7b8bd8a3d8f91abb3f89, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, c15775553a47462f4ada40853ab24526, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:05,927 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/.tmp/testFamily/d07ab35c2d0c4268becf0967a28b990c is 61, key is row-0/testFamily:testQualifier/1704100344546/Put/seqid=0 2024-01-01T09:13:05,933 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/info/0d1f0696bcee4b4093e8f7b4f4781629 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4./info:regioninfo/1704100197291/Put/seqid=0 2024-01-01T09:13:05,943 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/.tmp/testFamily/d07ab35c2d0c4268becf0967a28b990c 2024-01-01T09:13:05,991 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/.tmp/testFamily/d07ab35c2d0c4268becf0967a28b990c as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/testFamily/d07ab35c2d0c4268becf0967a28b990c 2024-01-01T09:13:05,997 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/testFamily/d07ab35c2d0c4268becf0967a28b990c, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:06,003 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9033d95dea4b7b8bd8a3d8f91abb3f89 in 135ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:06,008 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_1_-true/9033d95dea4b7b8bd8a3d8f91abb3f89/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:06,010 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9033d95dea4b7b8bd8a3d8f91abb3f89: 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1704100341527.9033d95dea4b7b8bd8a3d8f91abb3f89. 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 07df45817a4b1a315f53d0436e57ec0e, disabling compactions & flushes 2024-01-01T09:13:06,010 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. after waiting 0 ms 2024-01-01T09:13:06,010 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:13:06,010 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 07df45817a4b1a315f53d0436e57ec0e 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:06,024 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/.tmp/testFamily/5f46dda8ec314f3a9b44e320a92be499 is 61, key is row-0/testFamily:testQualifier/1704100288784/Put/seqid=0 2024-01-01T09:13:06,069 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 07df45817a4b1a315f53d0436e57ec0e, 1588230740, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, c15775553a47462f4ada40853ab24526, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:06,269 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 07df45817a4b1a315f53d0436e57ec0e, 1588230740, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, c15775553a47462f4ada40853ab24526, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:06,344 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.45 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/info/0d1f0696bcee4b4093e8f7b4f4781629 2024-01-01T09:13:06,366 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/table/bcaf82e938c5433f9a9d1c150d8880e3 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1704100197303/Put/seqid=0 2024-01-01T09:13:06,373 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.5.jar:?] 2024-01-01T09:13:06,431 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/.tmp/testFamily/5f46dda8ec314f3a9b44e320a92be499 2024-01-01T09:13:06,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/.tmp/testFamily/5f46dda8ec314f3a9b44e320a92be499 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/testFamily/5f46dda8ec314f3a9b44e320a92be499 2024-01-01T09:13:06,443 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/testFamily/5f46dda8ec314f3a9b44e320a92be499, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:06,444 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 07df45817a4b1a315f53d0436e57ec0e in 434ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:06,448 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/07df45817a4b1a315f53d0436e57ec0e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:06,449 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 07df45817a4b1a315f53d0436e57ec0e: 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1704100285765.07df45817a4b1a315f53d0436e57ec0e. 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing c15775553a47462f4ada40853ab24526, disabling compactions & flushes 2024-01-01T09:13:06,449 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. after waiting 0 ms 2024-01-01T09:13:06,449 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:13:06,449 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing c15775553a47462f4ada40853ab24526 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:06,461 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/.tmp/testFamily/01cb097f0bb446758db1d1b5813a2550 is 61, key is row-0/testFamily:testQualifier/1704100261550/Put/seqid=0 2024-01-01T09:13:06,467 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/.tmp/testFamily/01cb097f0bb446758db1d1b5813a2550 2024-01-01T09:13:06,470 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 1588230740, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, c15775553a47462f4ada40853ab24526, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:06,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/.tmp/testFamily/01cb097f0bb446758db1d1b5813a2550 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/testFamily/01cb097f0bb446758db1d1b5813a2550 2024-01-01T09:13:06,479 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/testFamily/01cb097f0bb446758db1d1b5813a2550, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:06,480 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c15775553a47462f4ada40853ab24526 in 31ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:06,485 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/c15775553a47462f4ada40853ab24526/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:06,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for c15775553a47462f4ada40853ab24526: 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1704100258531.c15775553a47462f4ada40853ab24526. 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 48930314afc32fca9384abf242612a24, disabling compactions & flushes 2024-01-01T09:13:06,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. after waiting 0 ms 2024-01-01T09:13:06,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:06,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 48930314afc32fca9384abf242612a24 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:06,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/.tmp/testFamily/12e1bcde22894e4e9bb16aa4d4e71bd5 is 61, key is row-0/testFamily:testQualifier/1704100318287/Put/seqid=0 2024-01-01T09:13:06,649 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-01-01T09:13:06,649 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-01-01T09:13:06,670 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 1588230740, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:06,676 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-01-01T09:13:06,773 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/table/bcaf82e938c5433f9a9d1c150d8880e3 2024-01-01T09:13:06,780 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/info/0d1f0696bcee4b4093e8f7b4f4781629 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/info/0d1f0696bcee4b4093e8f7b4f4781629 2024-01-01T09:13:06,787 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/info/0d1f0696bcee4b4093e8f7b4f4781629, entries=290, sequenceid=149, filesize=51.4 K 2024-01-01T09:13:06,788 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/.tmp/table/bcaf82e938c5433f9a9d1c150d8880e3 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/table/bcaf82e938c5433f9a9d1c150d8880e3 2024-01-01T09:13:06,794 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/table/bcaf82e938c5433f9a9d1c150d8880e3, entries=58, sequenceid=149, filesize=9.6 K 2024-01-01T09:13:06,795 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~48.69 KB/49854, heapSize ~74.80 KB/76592, currentSize=0 B/0 for 1588230740 in 927ms, sequenceid=149, compaction requested=false 2024-01-01T09:13:06,800 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-01-01T09:13:06,801 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-01-01T09:13:06,801 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2024-01-01T09:13:06,801 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2024-01-01T09:13:06,802 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-01-01T09:13:06,870 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 26 regions to close 2024-01-01T09:13:06,870 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 48930314afc32fca9384abf242612a24, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:06,904 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/.tmp/testFamily/12e1bcde22894e4e9bb16aa4d4e71bd5 2024-01-01T09:13:06,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/.tmp/testFamily/12e1bcde22894e4e9bb16aa4d4e71bd5 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/testFamily/12e1bcde22894e4e9bb16aa4d4e71bd5 2024-01-01T09:13:06,917 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/testFamily/12e1bcde22894e4e9bb16aa4d4e71bd5, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:06,918 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 48930314afc32fca9384abf242612a24 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:06,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/48930314afc32fca9384abf242612a24/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:06,923 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 48930314afc32fca9384abf242612a24: 2024-01-01T09:13:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1704100317273.48930314afc32fca9384abf242612a24. 2024-01-01T09:13:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0295370e633733ae7988ee9d3ebc2908, disabling compactions & flushes 2024-01-01T09:13:06,923 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:13:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:13:06,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. after waiting 0 ms 2024-01-01T09:13:06,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:13:06,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 0295370e633733ae7988ee9d3ebc2908 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:06,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/.tmp/testFamily/22c8f4da263e417fbde4d83b5b5429ad is 61, key is row-0/testFamily:testQualifier/1704100330423/Put/seqid=0 2024-01-01T09:13:07,071 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:07,271 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 0295370e633733ae7988ee9d3ebc2908, 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:07,342 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/.tmp/testFamily/22c8f4da263e417fbde4d83b5b5429ad 2024-01-01T09:13:07,349 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/.tmp/testFamily/22c8f4da263e417fbde4d83b5b5429ad as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/testFamily/22c8f4da263e417fbde4d83b5b5429ad 2024-01-01T09:13:07,355 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/testFamily/22c8f4da263e417fbde4d83b5b5429ad, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:07,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0295370e633733ae7988ee9d3ebc2908 in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:07,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_1_-true/0295370e633733ae7988ee9d3ebc2908/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:07,362 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:13:07,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0295370e633733ae7988ee9d3ebc2908: 2024-01-01T09:13:07,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1704100327402.0295370e633733ae7988ee9d3ebc2908. 2024-01-01T09:13:07,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 415f19567bc32def48eed0666b8f376c, disabling compactions & flushes 2024-01-01T09:13:07,363 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:13:07,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:13:07,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. after waiting 0 ms 2024-01-01T09:13:07,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:13:07,363 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 415f19567bc32def48eed0666b8f376c 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:07,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/.tmp/testFamily/475667d3bac2498db619109a32898130 is 61, key is row-0/testFamily:testQualifier/1704100291865/Put/seqid=0 2024-01-01T09:13:07,471 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:07,671 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 415f19567bc32def48eed0666b8f376c, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:07,705 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:13:07,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/.tmp/testFamily/475667d3bac2498db619109a32898130 2024-01-01T09:13:07,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/.tmp/testFamily/475667d3bac2498db619109a32898130 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/testFamily/475667d3bac2498db619109a32898130 2024-01-01T09:13:07,799 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/testFamily/475667d3bac2498db619109a32898130, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:07,800 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 415f19567bc32def48eed0666b8f376c in 437ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:07,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/415f19567bc32def48eed0666b8f376c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:07,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:13:07,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 415f19567bc32def48eed0666b8f376c: 2024-01-01T09:13:07,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1704100288845.415f19567bc32def48eed0666b8f376c. 2024-01-01T09:13:07,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9a4fa884b1f0e010faf65e75081899ff, disabling compactions & flushes 2024-01-01T09:13:07,806 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:13:07,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:13:07,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. after waiting 0 ms 2024-01-01T09:13:07,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:13:07,806 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9a4fa884b1f0e010faf65e75081899ff 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:07,818 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/.tmp/testFamily/b41d13ba387c4b089e2266bb389e4345 is 61, key is row-0/testFamily:testQualifier/1704100251466/Put/seqid=0 2024-01-01T09:13:07,872 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 23 regions to close 2024-01-01T09:13:07,872 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:07,889 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(536): Scanner lease -6882434854160433017 expired clientIPAndPort=172.31.12.81:36878, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:08,072 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, 9a4fa884b1f0e010faf65e75081899ff, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:08,224 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/.tmp/testFamily/b41d13ba387c4b089e2266bb389e4345 2024-01-01T09:13:08,231 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/.tmp/testFamily/b41d13ba387c4b089e2266bb389e4345 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/testFamily/b41d13ba387c4b089e2266bb389e4345 2024-01-01T09:13:08,237 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/testFamily/b41d13ba387c4b089e2266bb389e4345, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:08,238 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9a4fa884b1f0e010faf65e75081899ff in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:08,242 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/9a4fa884b1f0e010faf65e75081899ff/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:08,243 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:13:08,243 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9a4fa884b1f0e010faf65e75081899ff: 2024-01-01T09:13:08,243 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1704100250447.9a4fa884b1f0e010faf65e75081899ff. 2024-01-01T09:13:08,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 765fa36cba65fd597c2600619705e027, disabling compactions & flushes 2024-01-01T09:13:08,244 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:13:08,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:13:08,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. after waiting 0 ms 2024-01-01T09:13:08,244 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:13:08,244 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 765fa36cba65fd597c2600619705e027 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:08,256 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/.tmp/testFamily/bf3a789371d6492c871e12a1cf646922 is 61, key is row-0/testFamily:testQualifier/1704100301934/Put/seqid=0 2024-01-01T09:13:08,272 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:08,472 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:08,663 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/.tmp/testFamily/bf3a789371d6492c871e12a1cf646922 2024-01-01T09:13:08,669 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/.tmp/testFamily/bf3a789371d6492c871e12a1cf646922 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/testFamily/bf3a789371d6492c871e12a1cf646922 2024-01-01T09:13:08,673 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 765fa36cba65fd597c2600619705e027, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:08,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/testFamily/bf3a789371d6492c871e12a1cf646922, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:08,676 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 765fa36cba65fd597c2600619705e027 in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:08,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/765fa36cba65fd597c2600619705e027/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:08,681 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 765fa36cba65fd597c2600619705e027: 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1704100300917.765fa36cba65fd597c2600619705e027. 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2f6c92576d2ad95a8dd660702ba0381a, disabling compactions & flushes 2024-01-01T09:13:08,681 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. after waiting 0 ms 2024-01-01T09:13:08,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:13:08,682 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2f6c92576d2ad95a8dd660702ba0381a 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:08,693 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/.tmp/testFamily/ceb07a77018e404a97150120f4401d43 is 61, key is row-0/testFamily:testQualifier/1704100315214/Put/seqid=0 2024-01-01T09:13:08,873 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 21 regions to close 2024-01-01T09:13:08,873 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,073 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 2f6c92576d2ad95a8dd660702ba0381a, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/.tmp/testFamily/ceb07a77018e404a97150120f4401d43 2024-01-01T09:13:09,106 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/.tmp/testFamily/ceb07a77018e404a97150120f4401d43 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/testFamily/ceb07a77018e404a97150120f4401d43 2024-01-01T09:13:09,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/testFamily/ceb07a77018e404a97150120f4401d43, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:09,113 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2f6c92576d2ad95a8dd660702ba0381a in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:09,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_1_-true/2f6c92576d2ad95a8dd660702ba0381a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:09,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2f6c92576d2ad95a8dd660702ba0381a: 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1704100314197.2f6c92576d2ad95a8dd660702ba0381a. 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1db8a2d69b3d0f098a78503fcb88cb43, disabling compactions & flushes 2024-01-01T09:13:09,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. after waiting 0 ms 2024-01-01T09:13:09,119 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:13:09,119 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 1db8a2d69b3d0f098a78503fcb88cb43 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:09,131 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/.tmp/testFamily/5e4e5c446bb6426b996bce371da2e503 is 61, key is row-0/testFamily:testQualifier/1704100223965/Put/seqid=0 2024-01-01T09:13:09,273 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,474 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 1db8a2d69b3d0f098a78503fcb88cb43, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/.tmp/testFamily/5e4e5c446bb6426b996bce371da2e503 2024-01-01T09:13:09,546 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/.tmp/testFamily/5e4e5c446bb6426b996bce371da2e503 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/testFamily/5e4e5c446bb6426b996bce371da2e503 2024-01-01T09:13:09,552 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/testFamily/5e4e5c446bb6426b996bce371da2e503, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:09,553 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1db8a2d69b3d0f098a78503fcb88cb43 in 434ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:09,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnRenewLease_0_-false/1db8a2d69b3d0f098a78503fcb88cb43/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:09,558 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:13:09,558 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1db8a2d69b3d0f098a78503fcb88cb43: 2024-01-01T09:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1704100220943.1db8a2d69b3d0f098a78503fcb88cb43. 2024-01-01T09:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1c9c1fffb4d91d95f7adedde98457b0a, disabling compactions & flushes 2024-01-01T09:13:09,559 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. after waiting 0 ms 2024-01-01T09:13:09,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:13:09,559 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 1c9c1fffb4d91d95f7adedde98457b0a 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:09,571 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/.tmp/testFamily/6c3d42e38ad841368fec3912e5260f9c is 61, key is row-0/testFamily:testQualifier/1704100237233/Put/seqid=0 2024-01-01T09:13:09,674 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,874 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 19 regions to close 2024-01-01T09:13:09,874 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 1c9c1fffb4d91d95f7adedde98457b0a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:09,977 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/.tmp/testFamily/6c3d42e38ad841368fec3912e5260f9c 2024-01-01T09:13:09,983 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/.tmp/testFamily/6c3d42e38ad841368fec3912e5260f9c as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/testFamily/6c3d42e38ad841368fec3912e5260f9c 2024-01-01T09:13:09,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/testFamily/6c3d42e38ad841368fec3912e5260f9c, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:09,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1c9c1fffb4d91d95f7adedde98457b0a in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:09,994 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnClose_0_-false/1c9c1fffb4d91d95f7adedde98457b0a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:09,995 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:13:09,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1c9c1fffb4d91d95f7adedde98457b0a: 2024-01-01T09:13:09,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1704100236212.1c9c1fffb4d91d95f7adedde98457b0a. 2024-01-01T09:13:09,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 66680db6b4f1ced72c27dcba496ed9e4, disabling compactions & flushes 2024-01-01T09:13:09,996 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:09,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:09,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. after waiting 0 ms 2024-01-01T09:13:09,996 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:09,996 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 66680db6b4f1ced72c27dcba496ed9e4 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:10,009 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/.tmp/testFamily/ba60453c7b4f4d1d8f13ba16ef98ebc2 is 61, key is row-0/testFamily:testQualifier/1704100324351/Put/seqid=0 2024-01-01T09:13:10,074 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:10,275 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 66680db6b4f1ced72c27dcba496ed9e4, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:10,414 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/.tmp/testFamily/ba60453c7b4f4d1d8f13ba16ef98ebc2 2024-01-01T09:13:10,421 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/.tmp/testFamily/ba60453c7b4f4d1d8f13ba16ef98ebc2 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/testFamily/ba60453c7b4f4d1d8f13ba16ef98ebc2 2024-01-01T09:13:10,427 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/testFamily/ba60453c7b4f4d1d8f13ba16ef98ebc2, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:10,428 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66680db6b4f1ced72c27dcba496ed9e4 in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:10,433 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_1_-true/66680db6b4f1ced72c27dcba496ed9e4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:10,434 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 66680db6b4f1ced72c27dcba496ed9e4: 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1704100321334.66680db6b4f1ced72c27dcba496ed9e4. 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 84af1702ea2d10a3f3366f7bcc7c8e30, disabling compactions & flushes 2024-01-01T09:13:10,434 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. after waiting 0 ms 2024-01-01T09:13:10,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:13:10,434 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 84af1702ea2d10a3f3366f7bcc7c8e30 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:10,447 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/.tmp/testFamily/a73b3e7a6a1146c4b0d0d3f454acbf48 is 61, key is row-0/testFamily:testQualifier/1704100198696/Put/seqid=0 2024-01-01T09:13:10,475 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:10,675 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 84af1702ea2d10a3f3366f7bcc7c8e30, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:10,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/.tmp/testFamily/a73b3e7a6a1146c4b0d0d3f454acbf48 2024-01-01T09:13:10,860 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/.tmp/testFamily/a73b3e7a6a1146c4b0d0d3f454acbf48 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/testFamily/a73b3e7a6a1146c4b0d0d3f454acbf48 2024-01-01T09:13:10,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/testFamily/a73b3e7a6a1146c4b0d0d3f454acbf48, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:10,866 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 84af1702ea2d10a3f3366f7bcc7c8e30 in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:10,870 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/84af1702ea2d10a3f3366f7bcc7c8e30/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:10,871 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:13:10,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 84af1702ea2d10a3f3366f7bcc7c8e30: 2024-01-01T09:13:10,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1704100197675.84af1702ea2d10a3f3366f7bcc7c8e30. 2024-01-01T09:13:10,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 44e29dca8b8d85e87b669d8fa8e89f6f, disabling compactions & flushes 2024-01-01T09:13:10,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:13:10,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:13:10,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. after waiting 0 ms 2024-01-01T09:13:10,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:13:10,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 44e29dca8b8d85e87b669d8fa8e89f6f 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:10,875 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 16 regions to close 2024-01-01T09:13:10,875 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:10,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/.tmp/testFamily/4b05c01b2f74495f96202f4890e260f7 is 61, key is row-0/testFamily:testQualifier/1704100360701/Put/seqid=0 2024-01-01T09:13:11,075 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:11,276 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e29dca8b8d85e87b669d8fa8e89f6f, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:11,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/.tmp/testFamily/4b05c01b2f74495f96202f4890e260f7 2024-01-01T09:13:11,296 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/.tmp/testFamily/4b05c01b2f74495f96202f4890e260f7 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/testFamily/4b05c01b2f74495f96202f4890e260f7 2024-01-01T09:13:11,301 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/testFamily/4b05c01b2f74495f96202f4890e260f7, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:11,302 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 44e29dca8b8d85e87b669d8fa8e89f6f in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:11,307 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/44e29dca8b8d85e87b669d8fa8e89f6f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:11,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 44e29dca8b8d85e87b669d8fa8e89f6f: 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1704100357683.44e29dca8b8d85e87b669d8fa8e89f6f. 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f4a3b78bdc31e990f7f520db112b2f63, disabling compactions & flushes 2024-01-01T09:13:11,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. after waiting 0 ms 2024-01-01T09:13:11,308 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:13:11,308 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing f4a3b78bdc31e990f7f520db112b2f63 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:11,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/.tmp/testFamily/ad686a0b623d492898418f83481f9f30 is 61, key is row-0/testFamily:testQualifier/1704100314009/Put/seqid=0 2024-01-01T09:13:11,476 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:11,676 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, f4a3b78bdc31e990f7f520db112b2f63, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:11,727 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/.tmp/testFamily/ad686a0b623d492898418f83481f9f30 2024-01-01T09:13:11,733 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/.tmp/testFamily/ad686a0b623d492898418f83481f9f30 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/testFamily/ad686a0b623d492898418f83481f9f30 2024-01-01T09:13:11,738 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/testFamily/ad686a0b623d492898418f83481f9f30, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:11,739 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f4a3b78bdc31e990f7f520db112b2f63 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:11,743 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_1_-true/f4a3b78bdc31e990f7f520db112b2f63/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:11,744 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:13:11,744 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f4a3b78bdc31e990f7f520db112b2f63: 2024-01-01T09:13:11,744 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1704100310991.f4a3b78bdc31e990f7f520db112b2f63. 2024-01-01T09:13:11,745 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9524465155bd6327e9464a3a91161d37, disabling compactions & flushes 2024-01-01T09:13:11,745 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:13:11,745 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:13:11,745 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. after waiting 0 ms 2024-01-01T09:13:11,745 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:13:11,745 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9524465155bd6327e9464a3a91161d37 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:11,757 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/.tmp/testFamily/3988fe84c194495e8d822f3f57632507 is 61, key is row-0/testFamily:testQualifier/1704100339984/Put/seqid=0 2024-01-01T09:13:11,876 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 14 regions to close 2024-01-01T09:13:11,876 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:12,077 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, 9524465155bd6327e9464a3a91161d37, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:12,162 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/.tmp/testFamily/3988fe84c194495e8d822f3f57632507 2024-01-01T09:13:12,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/.tmp/testFamily/3988fe84c194495e8d822f3f57632507 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/testFamily/3988fe84c194495e8d822f3f57632507 2024-01-01T09:13:12,174 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/testFamily/3988fe84c194495e8d822f3f57632507, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:12,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9524465155bd6327e9464a3a91161d37 in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:12,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_1_-true/9524465155bd6327e9464a3a91161d37/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:12,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9524465155bd6327e9464a3a91161d37: 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1704100336966.9524465155bd6327e9464a3a91161d37. 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5c3468183b42b1b15d589cef672ad7b1, disabling compactions & flushes 2024-01-01T09:13:12,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. after waiting 0 ms 2024-01-01T09:13:12,181 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:13:12,181 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 5c3468183b42b1b15d589cef672ad7b1 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:12,194 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/.tmp/testFamily/76b4353d1e094c67b5a832ca0986e94f is 61, key is row-0/testFamily:testQualifier/1704100220873/Put/seqid=0 2024-01-01T09:13:12,277 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:12,443 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(140): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=jenkins-hbase3.apache.org:46807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:217) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:394) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:335) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:93) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:599) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.getLiveRegionServers(RegionServerStatusProtos.java:17993) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionImplementation.getLiveRegionServers(ConnectionImplementation.java:2375) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:138) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:365) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:447) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 13 more 2024-01-01T09:13:12,477 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 5c3468183b42b1b15d589cef672ad7b1, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:12,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/.tmp/testFamily/76b4353d1e094c67b5a832ca0986e94f 2024-01-01T09:13:12,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/.tmp/testFamily/76b4353d1e094c67b5a832ca0986e94f as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/testFamily/76b4353d1e094c67b5a832ca0986e94f 2024-01-01T09:13:12,611 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/testFamily/76b4353d1e094c67b5a832ca0986e94f, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:12,612 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5c3468183b42b1b15d589cef672ad7b1 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:12,617 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextException_0_-false/5c3468183b42b1b15d589cef672ad7b1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:12,617 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:13:12,617 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5c3468183b42b1b15d589cef672ad7b1: 2024-01-01T09:13:12,617 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1704100217847.5c3468183b42b1b15d589cef672ad7b1. 2024-01-01T09:13:12,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2cb844b5df2310f6a3c3ae750e9d60fb, disabling compactions & flushes 2024-01-01T09:13:12,618 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:13:12,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:13:12,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. after waiting 0 ms 2024-01-01T09:13:12,618 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:13:12,618 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2cb844b5df2310f6a3c3ae750e9d60fb 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:12,630 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/.tmp/testFamily/d19d31ed3de94465a5c66bb694dced01 is 61, key is row-0/testFamily:testQualifier/1704100249376/Put/seqid=0 2024-01-01T09:13:12,677 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:12,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:13:12,877 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 12 regions to close 2024-01-01T09:13:12,877 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 2cb844b5df2310f6a3c3ae750e9d60fb, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,036 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/.tmp/testFamily/d19d31ed3de94465a5c66bb694dced01 2024-01-01T09:13:13,042 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/.tmp/testFamily/d19d31ed3de94465a5c66bb694dced01 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/testFamily/d19d31ed3de94465a5c66bb694dced01 2024-01-01T09:13:13,048 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/testFamily/d19d31ed3de94465a5c66bb694dced01, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:13,049 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2cb844b5df2310f6a3c3ae750e9d60fb in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:13,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNext_0_-false/2cb844b5df2310f6a3c3ae750e9d60fb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:13,055 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:13:13,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2cb844b5df2310f6a3c3ae750e9d60fb: 2024-01-01T09:13:13,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1704100248358.2cb844b5df2310f6a3c3ae750e9d60fb. 2024-01-01T09:13:13,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2c9944b1c50d845091a1dba2725ec2ba, disabling compactions & flushes 2024-01-01T09:13:13,056 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:13:13,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:13:13,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. after waiting 0 ms 2024-01-01T09:13:13,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:13:13,056 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2c9944b1c50d845091a1dba2725ec2ba 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:13,069 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/.tmp/testFamily/cd54c39ecf7e4d9ca28419fef186c63e is 61, key is row-0/testFamily:testQualifier/1704100350620/Put/seqid=0 2024-01-01T09:13:13,078 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-01-01T09:13:13,149 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-01-01T09:13:13,150 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-01-01T09:13:13,278 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/.tmp/testFamily/cd54c39ecf7e4d9ca28419fef186c63e 2024-01-01T09:13:13,478 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 2c9944b1c50d845091a1dba2725ec2ba, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,480 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/.tmp/testFamily/cd54c39ecf7e4d9ca28419fef186c63e as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/testFamily/cd54c39ecf7e4d9ca28419fef186c63e 2024-01-01T09:13:13,486 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/testFamily/cd54c39ecf7e4d9ca28419fef186c63e, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:13,488 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2c9944b1c50d845091a1dba2725ec2ba in 432ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:13,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/2c9944b1c50d845091a1dba2725ec2ba/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:13,495 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2c9944b1c50d845091a1dba2725ec2ba: 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1704100347601.2c9944b1c50d845091a1dba2725ec2ba. 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fd7437d2995aea437ed3772b0263c069, disabling compactions & flushes 2024-01-01T09:13:13,495 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. after waiting 0 ms 2024-01-01T09:13:13,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:13:13,495 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing fd7437d2995aea437ed3772b0263c069 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:13,510 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/.tmp/testFamily/b874df078a2c418abfa932fdff9ee7f5 is 61, key is row-0/testFamily:testQualifier/1704100372769/Put/seqid=0 2024-01-01T09:13:13,678 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,878 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 10 regions to close 2024-01-01T09:13:13,879 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4, fd7437d2995aea437ed3772b0263c069 2024-01-01T09:13:13,916 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/.tmp/testFamily/b874df078a2c418abfa932fdff9ee7f5 2024-01-01T09:13:13,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/.tmp/testFamily/b874df078a2c418abfa932fdff9ee7f5 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/testFamily/b874df078a2c418abfa932fdff9ee7f5 2024-01-01T09:13:13,928 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/testFamily/b874df078a2c418abfa932fdff9ee7f5, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:13,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fd7437d2995aea437ed3772b0263c069 in 434ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:13,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_1_-true/fd7437d2995aea437ed3772b0263c069/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:13,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:13:13,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fd7437d2995aea437ed3772b0263c069: 2024-01-01T09:13:13,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1704100369752.fd7437d2995aea437ed3772b0263c069. 2024-01-01T09:13:13,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b12eec449640ec3fe233eaeeed122249, disabling compactions & flushes 2024-01-01T09:13:13,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:13:13,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:13:13,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. after waiting 0 ms 2024-01-01T09:13:13,935 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:13:13,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing b12eec449640ec3fe233eaeeed122249 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:13,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/.tmp/testFamily/632311fa20334c1c90e4662561e8acc4 is 61, key is row-0/testFamily:testQualifier/1704100246804/Put/seqid=0 2024-01-01T09:13:14,079 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:13:14,279 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, b12eec449640ec3fe233eaeeed122249, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:13:14,352 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/.tmp/testFamily/632311fa20334c1c90e4662561e8acc4 2024-01-01T09:13:14,358 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/.tmp/testFamily/632311fa20334c1c90e4662561e8acc4 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/testFamily/632311fa20334c1c90e4662561e8acc4 2024-01-01T09:13:14,364 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/testFamily/632311fa20334c1c90e4662561e8acc4, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:14,365 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b12eec449640ec3fe233eaeeed122249 in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:14,369 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnClose_0_-false/b12eec449640ec3fe233eaeeed122249/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:14,369 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b12eec449640ec3fe233eaeeed122249: 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1704100243782.b12eec449640ec3fe233eaeeed122249. 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fb7a1d2a94462bd3dc4f09bfab6db5a4, disabling compactions & flushes 2024-01-01T09:13:14,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. after waiting 0 ms 2024-01-01T09:13:14,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:13:14,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing fb7a1d2a94462bd3dc4f09bfab6db5a4 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:14,382 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/.tmp/testFamily/cf3121d0e7074a80b8607e81b459d439 is 61, key is row-0/testFamily:testQualifier/1704100197547/Put/seqid=0 2024-01-01T09:13:14,479 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:13:14,679 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3, fb7a1d2a94462bd3dc4f09bfab6db5a4 2024-01-01T09:13:14,787 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/.tmp/testFamily/cf3121d0e7074a80b8607e81b459d439 2024-01-01T09:13:14,793 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/.tmp/testFamily/cf3121d0e7074a80b8607e81b459d439 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/testFamily/cf3121d0e7074a80b8607e81b459d439 2024-01-01T09:13:14,798 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/testFamily/cf3121d0e7074a80b8607e81b459d439, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:14,799 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fb7a1d2a94462bd3dc4f09bfab6db5a4 in 429ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/fb7a1d2a94462bd3dc4f09bfab6db5a4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:14,804 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fb7a1d2a94462bd3dc4f09bfab6db5a4: 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1704100196496.fb7a1d2a94462bd3dc4f09bfab6db5a4. 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 029d4c0e35227b01f80bf9b14d52df3a, disabling compactions & flushes 2024-01-01T09:13:14,804 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. after waiting 0 ms 2024-01-01T09:13:14,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:13:14,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 029d4c0e35227b01f80bf9b14d52df3a 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:14,816 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/.tmp/testFamily/044c4d97d26d4573a7a1265f740b9c08 is 61, key is row-0/testFamily:testQualifier/1704100208779/Put/seqid=0 2024-01-01T09:13:14,880 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 7 regions to close 2024-01-01T09:13:14,880 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1599): Online Regions={029d4c0e35227b01f80bf9b14d52df3a=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a., 44e98b595fab849d235acbb2941a22f8=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8., 8b769a5133bcc30546aabb30cc7e069f=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., 345536859393f3bdc499b7c4b1b8bb53=testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53., 8fe5994cdb848b90b5ea2670beebb22f=hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f., ca7ad7cffa9e545916841f9a00e288f3=hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3., 257e5f4c014f577810ee0791199ada41=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.} 2024-01-01T09:13:14,880 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:15,080 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 029d4c0e35227b01f80bf9b14d52df3a, 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:15,221 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/.tmp/testFamily/044c4d97d26d4573a7a1265f740b9c08 2024-01-01T09:13:15,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/.tmp/testFamily/044c4d97d26d4573a7a1265f740b9c08 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/testFamily/044c4d97d26d4573a7a1265f740b9c08 2024-01-01T09:13:15,233 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/testFamily/044c4d97d26d4573a7a1265f740b9c08, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:15,234 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 029d4c0e35227b01f80bf9b14d52df3a in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/029d4c0e35227b01f80bf9b14d52df3a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:15,238 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 029d4c0e35227b01f80bf9b14d52df3a: 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1704100207758.029d4c0e35227b01f80bf9b14d52df3a. 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 44e98b595fab849d235acbb2941a22f8, disabling compactions & flushes 2024-01-01T09:13:15,238 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. after waiting 0 ms 2024-01-01T09:13:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:15,238 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 44e98b595fab849d235acbb2941a22f8 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:15,250 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/.tmp/testFamily/b76304cd1799405a95d92f2831581df5 is 61, key is row-0/testFamily:testQualifier/1704100382838/Put/seqid=0 2024-01-01T09:13:15,280 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:15,480 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 44e98b595fab849d235acbb2941a22f8, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:15,656 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/.tmp/testFamily/b76304cd1799405a95d92f2831581df5 2024-01-01T09:13:15,661 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/.tmp/testFamily/b76304cd1799405a95d92f2831581df5 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/testFamily/b76304cd1799405a95d92f2831581df5 2024-01-01T09:13:15,667 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/testFamily/b76304cd1799405a95d92f2831581df5, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:15,667 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 44e98b595fab849d235acbb2941a22f8 in 429ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:15,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/44e98b595fab849d235acbb2941a22f8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:15,672 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:15,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 44e98b595fab849d235acbb2941a22f8: 2024-01-01T09:13:15,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1704100381821.44e98b595fab849d235acbb2941a22f8. 2024-01-01T09:13:15,672 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8b769a5133bcc30546aabb30cc7e069f, disabling compactions & flushes 2024-01-01T09:13:15,673 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:13:15,673 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:13:15,673 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. after waiting 0 ms 2024-01-01T09:13:15,673 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:13:15,673 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8b769a5133bcc30546aabb30cc7e069f 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:15,680 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:15,685 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/.tmp/testFamily/7c62b930c2a749258fa22c6b3a36e299 is 61, key is row-0/testFamily:testQualifier/1704100227059/Put/seqid=0 2024-01-01T09:13:15,881 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 5 regions to close 2024-01-01T09:13:15,881 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1599): Online Regions={8b769a5133bcc30546aabb30cc7e069f=testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f., 345536859393f3bdc499b7c4b1b8bb53=testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53., 8fe5994cdb848b90b5ea2670beebb22f=hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f., ca7ad7cffa9e545916841f9a00e288f3=hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3., 257e5f4c014f577810ee0791199ada41=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.} 2024-01-01T09:13:15,881 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,081 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 8b769a5133bcc30546aabb30cc7e069f, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,090 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/.tmp/testFamily/7c62b930c2a749258fa22c6b3a36e299 2024-01-01T09:13:16,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/.tmp/testFamily/7c62b930c2a749258fa22c6b3a36e299 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/testFamily/7c62b930c2a749258fa22c6b3a36e299 2024-01-01T09:13:16,102 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/testFamily/7c62b930c2a749258fa22c6b3a36e299, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:16,103 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b769a5133bcc30546aabb30cc7e069f in 430ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:16,107 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/8b769a5133bcc30546aabb30cc7e069f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:16,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8b769a5133bcc30546aabb30cc7e069f: 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1704100226039.8b769a5133bcc30546aabb30cc7e069f. 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 345536859393f3bdc499b7c4b1b8bb53, disabling compactions & flushes 2024-01-01T09:13:16,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. after waiting 0 ms 2024-01-01T09:13:16,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:13:16,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 345536859393f3bdc499b7c4b1b8bb53 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:16,120 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/.tmp/testFamily/bba8007ab4554c1e8dc30a7b173dfad3 is 61, key is row-0/testFamily:testQualifier/1704100233141/Put/seqid=0 2024-01-01T09:13:16,281 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,481 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 345536859393f3bdc499b7c4b1b8bb53, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,526 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/.tmp/testFamily/bba8007ab4554c1e8dc30a7b173dfad3 2024-01-01T09:13:16,532 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/.tmp/testFamily/bba8007ab4554c1e8dc30a7b173dfad3 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/testFamily/bba8007ab4554c1e8dc30a7b173dfad3 2024-01-01T09:13:16,538 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/testFamily/bba8007ab4554c1e8dc30a7b173dfad3, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:16,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 345536859393f3bdc499b7c4b1b8bb53 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:16,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnNextAsync_0_-false/345536859393f3bdc499b7c4b1b8bb53/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:16,544 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 345536859393f3bdc499b7c4b1b8bb53: 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1704100230120.345536859393f3bdc499b7c4b1b8bb53. 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8fe5994cdb848b90b5ea2670beebb22f, disabling compactions & flushes 2024-01-01T09:13:16,544 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. after waiting 0 ms 2024-01-01T09:13:16,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:13:16,544 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8fe5994cdb848b90b5ea2670beebb22f 1/1 column families, dataSize=78 B heapSize=488 B 2024-01-01T09:13:16,556 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/.tmp/info/d5eaf720bf63446ab5e5af050943c629 is 45, key is default/info:d/1704100196236/Put/seqid=0 2024-01-01T09:13:16,682 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,882 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1595): Waiting on 3 regions to close 2024-01-01T09:13:16,882 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1599): Online Regions={8fe5994cdb848b90b5ea2670beebb22f=hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f., ca7ad7cffa9e545916841f9a00e288f3=hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3., 257e5f4c014f577810ee0791199ada41=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41.} 2024-01-01T09:13:16,882 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, 8fe5994cdb848b90b5ea2670beebb22f, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:16,961 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/.tmp/info/d5eaf720bf63446ab5e5af050943c629 2024-01-01T09:13:16,967 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/.tmp/info/d5eaf720bf63446ab5e5af050943c629 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/info/d5eaf720bf63446ab5e5af050943c629 2024-01-01T09:13:16,973 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/info/d5eaf720bf63446ab5e5af050943c629, entries=2, sequenceid=6, filesize=4.9 K 2024-01-01T09:13:16,974 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8fe5994cdb848b90b5ea2670beebb22f in 430ms, sequenceid=6, compaction requested=false 2024-01-01T09:13:16,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/namespace/8fe5994cdb848b90b5ea2670beebb22f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-01-01T09:13:16,978 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:13:16,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8fe5994cdb848b90b5ea2670beebb22f: 2024-01-01T09:13:16,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1704100195268.8fe5994cdb848b90b5ea2670beebb22f. 2024-01-01T09:13:16,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ca7ad7cffa9e545916841f9a00e288f3, disabling compactions & flushes 2024-01-01T09:13:16,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:13:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:13:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. after waiting 0 ms 2024-01-01T09:13:16,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:13:16,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ca7ad7cffa9e545916841f9a00e288f3 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:16,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/.tmp/testFamily/5604410d8f64403d82bb9d571cda6c41 is 61, key is row-0/testFamily:testQualifier/1704100273625/Put/seqid=0 2024-01-01T09:13:17,082 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:17,282 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41, ca7ad7cffa9e545916841f9a00e288f3 2024-01-01T09:13:17,398 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/.tmp/testFamily/5604410d8f64403d82bb9d571cda6c41 2024-01-01T09:13:17,404 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/.tmp/testFamily/5604410d8f64403d82bb9d571cda6c41 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/testFamily/5604410d8f64403d82bb9d571cda6c41 2024-01-01T09:13:17,409 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/testFamily/5604410d8f64403d82bb9d571cda6c41, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:17,410 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ca7ad7cffa9e545916841f9a00e288f3 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:17,414 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/hbase/testMetaScanTimeoutOnNext_0_-false/ca7ad7cffa9e545916841f9a00e288f3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:17,415 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ca7ad7cffa9e545916841f9a00e288f3: 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1704100270606.ca7ad7cffa9e545916841f9a00e288f3. 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 257e5f4c014f577810ee0791199ada41, disabling compactions & flushes 2024-01-01T09:13:17,415 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. after waiting 0 ms 2024-01-01T09:13:17,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:13:17,415 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 257e5f4c014f577810ee0791199ada41 1/1 column families, dataSize=228 B heapSize=800 B 2024-01-01T09:13:17,427 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/.tmp/testFamily/8a896a3fd5b4454fbcdf8082af7c65e2 is 61, key is row-0/testFamily:testQualifier/1704100282713/Put/seqid=0 2024-01-01T09:13:17,482 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41 2024-01-01T09:13:17,683 DEBUG [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1625): Waiting on 257e5f4c014f577810ee0791199ada41 2024-01-01T09:13:17,834 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/.tmp/testFamily/8a896a3fd5b4454fbcdf8082af7c65e2 2024-01-01T09:13:17,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/.tmp/testFamily/8a896a3fd5b4454fbcdf8082af7c65e2 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/testFamily/8a896a3fd5b4454fbcdf8082af7c65e2 2024-01-01T09:13:17,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/testFamily/8a896a3fd5b4454fbcdf8082af7c65e2, entries=4, sequenceid=8, filesize=4.9 K 2024-01-01T09:13:17,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 257e5f4c014f577810ee0791199ada41 in 431ms, sequenceid=8, compaction requested=false 2024-01-01T09:13:17,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/257e5f4c014f577810ee0791199ada41/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-01-01T09:13:17,851 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:13:17,852 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 257e5f4c014f577810ee0791199ada41: 2024-01-01T09:13:17,852 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1704100279695.257e5f4c014f577810ee0791199ada41. 2024-01-01T09:13:17,883 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1246): stopping server jenkins-hbase3.apache.org,44183,1704100133450; all regions closed. 2024-01-01T09:13:17,893 DEBUG [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/oldWALs 2024-01-01T09:13:17,893 INFO [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44183%2C1704100133450.meta:.meta(num 1704100195027) 2024-01-01T09:13:17,897 DEBUG [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(1062): Moved 1 WAL file(s) to /user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/oldWALs 2024-01-01T09:13:17,897 INFO [RS:0;jenkins-hbase3:44183 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44183%2C1704100133450:(num 1704100194708) 2024-01-01T09:13:17,897 DEBUG [RS:0;jenkins-hbase3:44183 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:17,897 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.LeaseManager(133): Closed leases 2024-01-01T09:13:17,898 INFO [RS:0;jenkins-hbase3:44183 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-01-01T09:13:17,898 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-01-01T09:13:17,899 INFO [RS:0;jenkins-hbase3:44183 {}] ipc.NettyRpcServer(348): Stopping server on /172.31.12.81:44183 2024-01-01T09:13:17,904 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase3.apache.org,44183,1704100133450 2024-01-01T09:13:17,904 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-01-01T09:13:17,906 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,44183,1704100133450] 2024-01-01T09:13:17,906 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,44183,1704100133450; numProcessing=1 2024-01-01T09:13:17,907 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase3.apache.org,44183,1704100133450 already deleted, retry=false 2024-01-01T09:13:17,907 INFO [RegionServerTracker-0 {}] master.ServerManager(562): Cluster shutdown set; jenkins-hbase3.apache.org,44183,1704100133450 expired; onlineServers=0 2024-01-01T09:13:17,907 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2554): ***** STOPPING region server 'jenkins-hbase3.apache.org,46807,1704100132244' ***** 2024-01-01T09:13:17,907 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2568): STOPPED: Cluster shutdown set; onlineServer=0 2024-01-01T09:13:17,908 DEBUG [M:0;jenkins-hbase3:46807 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3d157d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2024-01-01T09:13:17,908 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionServer(1220): stopping server jenkins-hbase3.apache.org,46807,1704100132244 2024-01-01T09:13:17,908 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionServer(1246): stopping server jenkins-hbase3.apache.org,46807,1704100132244; all regions closed. 2024-01-01T09:13:17,908 DEBUG [M:0;jenkins-hbase3:46807 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2024-01-01T09:13:17,908 DEBUG [M:0;jenkins-hbase3:46807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-01-01T09:13:17,908 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-01-01T09:13:17,908 DEBUG [M:0;jenkins-hbase3:46807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-01-01T09:13:17,908 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1704100134537 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1704100134537,5,FailOnTimeoutGroup] 2024-01-01T09:13:17,908 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1704100134537 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1704100134537,5,FailOnTimeoutGroup] 2024-01-01T09:13:17,908 INFO [M:0;jenkins-hbase3:46807 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2024-01-01T09:13:17,909 DEBUG [M:0;jenkins-hbase3:46807 {}] master.HMaster(1694): Stopping service threads 2024-01-01T09:13:17,909 INFO [M:0;jenkins-hbase3:46807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-01-01T09:13:17,909 ERROR [M:0;jenkins-hbase3:46807 {}] procedure2.ProcedureExecutor(701): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2024-01-01T09:13:17,910 INFO [M:0;jenkins-hbase3:46807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-01-01T09:13:17,910 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-01-01T09:13:17,912 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-01-01T09:13:17,912 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-01-01T09:13:17,912 DEBUG [M:0;jenkins-hbase3:46807 {}] zookeeper.ZKUtil(347): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-01-01T09:13:17,912 WARN [M:0;jenkins-hbase3:46807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-01-01T09:13:17,912 INFO [M:0;jenkins-hbase3:46807 {}] assignment.AssignmentManager(389): Stopping assignment manager 2024-01-01T09:13:17,912 INFO [M:0;jenkins-hbase3:46807 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-01-01T09:13:17,912 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-01-01T09:13:17,912 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-01-01T09:13:17,912 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:13:17,913 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:13:17,913 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-01-01T09:13:17,913 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:13:17,913 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=454.17 KB heapSize=525.63 KB 2024-01-01T09:13:17,927 DEBUG [M:0;jenkins-hbase3:46807 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63e966a8a93e401fa2e7e0f86ddb4ecd is 88, key is hbase:meta,,1/info:sn/1704100194796/Put/seqid=0 2024-01-01T09:13:17,997 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-01-01T09:13:18,006 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-01-01T09:13:18,006 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44183-0x1014e62600b0001, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-01-01T09:13:18,006 INFO [RS:0;jenkins-hbase3:44183 {}] regionserver.HRegionServer(1303): Exiting; stopping=jenkins-hbase3.apache.org,44183,1704100133450; zookeeper connection closed. 2024-01-01T09:13:18,007 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6319878 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6319878 2024-01-01T09:13:18,007 INFO [Listener at localhost/38993 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-01-01T09:13:18,119 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-01-01T09:13:18,335 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63e966a8a93e401fa2e7e0f86ddb4ecd 2024-01-01T09:13:18,354 DEBUG [M:0;jenkins-hbase3:46807 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b746b257d12d49748f7d5a5ab81ffd1b is 831, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1704100197312/Put/seqid=0 2024-01-01T09:13:18,760 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=453.58 KB at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b746b257d12d49748f7d5a5ab81ffd1b 2024-01-01T09:13:18,765 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b746b257d12d49748f7d5a5ab81ffd1b 2024-01-01T09:13:18,777 DEBUG [M:0;jenkins-hbase3:46807 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0acc4cc0d5d8456185cb36e348a31486 is 82, key is jenkins-hbase3.apache.org,44183,1704100133450/rs:state/1704100164578/Put/seqid=0 2024-01-01T09:13:19,182 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=830 (bloomFilter=true), to=hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0acc4cc0d5d8456185cb36e348a31486 2024-01-01T09:13:19,204 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/63e966a8a93e401fa2e7e0f86ddb4ecd as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63e966a8a93e401fa2e7e0f86ddb4ecd 2024-01-01T09:13:19,209 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/63e966a8a93e401fa2e7e0f86ddb4ecd, entries=8, sequenceid=830, filesize=5.5 K 2024-01-01T09:13:19,210 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b746b257d12d49748f7d5a5ab81ffd1b as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b746b257d12d49748f7d5a5ab81ffd1b 2024-01-01T09:13:19,214 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b746b257d12d49748f7d5a5ab81ffd1b 2024-01-01T09:13:19,214 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b746b257d12d49748f7d5a5ab81ffd1b, entries=92, sequenceid=830, filesize=29.6 K 2024-01-01T09:13:19,215 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0acc4cc0d5d8456185cb36e348a31486 as hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0acc4cc0d5d8456185cb36e348a31486 2024-01-01T09:13:19,219 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43719/user/jenkins/test-data/40e3a0e0-db83-6e1f-2432-bae691bbab86/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0acc4cc0d5d8456185cb36e348a31486, entries=1, sequenceid=830, filesize=5.1 K 2024-01-01T09:13:19,220 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(3022): Finished flush of dataSize ~454.17 KB/465074, heapSize ~525.33 KB/537936, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=830, compaction requested=false 2024-01-01T09:13:19,222 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-01-01T09:13:19,222 DEBUG [M:0;jenkins-hbase3:46807 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-01-01T09:13:19,225 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-01-01T09:13:19,225 INFO [M:0;jenkins-hbase3:46807 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2024-01-01T09:13:19,225 INFO [M:0;jenkins-hbase3:46807 {}] ipc.NettyRpcServer(348): Stopping server on /172.31.12.81:46807 2024-01-01T09:13:19,227 DEBUG [M:0;jenkins-hbase3:46807 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase3.apache.org,46807,1704100132244 already deleted, retry=false 2024-01-01T09:13:19,328 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-01-01T09:13:19,328 INFO [M:0;jenkins-hbase3:46807 {}] regionserver.HRegionServer(1303): Exiting; stopping=jenkins-hbase3.apache.org,46807,1704100132244; zookeeper connection closed. 2024-01-01T09:13:19,328 DEBUG [Listener at localhost/38993-EventThread {}] zookeeper.ZKWatcher(604): master:46807-0x1014e62600b0000, quorum=127.0.0.1:56255, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-01-01T09:13:19,330 WARN [Listener at localhost/38993 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-01-01T09:13:19,333 INFO [Listener at localhost/38993 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-01-01T09:13:19,435 WARN [BP-75454655-172.31.12.81-1704100129287 heartbeating to localhost/127.0.0.1:43719 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-01-01T09:13:19,435 WARN [BP-75454655-172.31.12.81-1704100129287 heartbeating to localhost/127.0.0.1:43719 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-75454655-172.31.12.81-1704100129287 (Datanode Uuid 0abf6536-c559-4747-8986-c6b584d20a6d) service to localhost/127.0.0.1:43719 2024-01-01T09:13:19,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864/dfs/data/data1/current/BP-75454655-172.31.12.81-1704100129287 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-01-01T09:13:19,437 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/16f47e11-17de-7f26-03bd-865160698c24/cluster_286652f6-1518-7d4c-d915-7f357e609864/dfs/data/data2/current/BP-75454655-172.31.12.81-1704100129287 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-01-01T09:13:19,457 INFO [Listener at localhost/38993 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-01-01T09:13:19,573 INFO [Listener at localhost/38993 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-01-01T09:13:19,614 INFO [Listener at localhost/38993 {}] hbase.HBaseTestingUtility(1347): Minicluster is down