2024-11-09 02:17:47,068 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-09 02:17:47,084 main DEBUG Took 0.013558 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 02:17:47,084 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 02:17:47,085 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 02:17:47,086 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 02:17:47,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,123 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 02:17:47,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,148 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,150 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,151 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,152 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,153 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,155 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:17:47,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,157 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 02:17:47,159 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:17:47,161 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 02:17:47,163 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 02:17:47,164 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 02:17:47,166 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 02:17:47,166 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 02:17:47,181 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 02:17:47,185 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 02:17:47,187 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 02:17:47,188 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 02:17:47,188 main DEBUG createAppenders(={Console}) 2024-11-09 02:17:47,189 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 initialized 2024-11-09 02:17:47,190 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 2024-11-09 02:17:47,190 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3f4faf53 OK. 2024-11-09 02:17:47,191 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 02:17:47,191 main DEBUG OutputStream closed 2024-11-09 02:17:47,192 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 02:17:47,192 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 02:17:47,193 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4d465b11 OK 2024-11-09 02:17:47,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 02:17:47,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 02:17:47,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 02:17:47,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 02:17:47,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 02:17:47,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 02:17:47,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 02:17:47,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 02:17:47,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 02:17:47,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 02:17:47,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 02:17:47,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 02:17:47,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 02:17:47,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 02:17:47,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 02:17:47,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 02:17:47,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 02:17:47,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 02:17:47,321 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 02:17:47,324 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@598bd2ba) with optional ClassLoader: null 2024-11-09 02:17:47,325 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 02:17:47,326 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@598bd2ba] started OK. 2024-11-09T02:17:47,646 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 26 mins 2024-11-09 02:17:47,652 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 02:17:47,653 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T02:17:47,827 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023 2024-11-09T02:17:47,881 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T02:17:47,885 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8, deleteOnExit=true 2024-11-09T02:17:47,885 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-09T02:17:47,886 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/test.cache.data in system properties and HBase conf 2024-11-09T02:17:47,886 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T02:17:47,888 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.log.dir in system properties and HBase conf 2024-11-09T02:17:47,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T02:17:47,889 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T02:17:47,890 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-09T02:17:48,014 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T02:17:48,184 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T02:17:48,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:17:48,190 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:17:48,191 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T02:17:48,192 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:17:48,193 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T02:17:48,193 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T02:17:48,194 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:17:48,194 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:17:48,195 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T02:17:48,196 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/nfs.dump.dir in system properties and HBase conf 2024-11-09T02:17:48,197 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/java.io.tmpdir in system properties and HBase conf 2024-11-09T02:17:48,197 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:17:48,198 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T02:17:48,198 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T02:17:49,353 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T02:17:49,463 INFO [Time-limited test {}] log.Log(170): Logging initialized @3421ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T02:17:49,562 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:17:49,650 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:17:49,687 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:17:49,687 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:17:49,689 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T02:17:49,709 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:17:49,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:17:49,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:17:50,004 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/java.io.tmpdir/jetty-localhost-36893-hadoop-hdfs-3_4_1-tests_jar-_-any-9236478231412019480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:17:50,012 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:36893} 2024-11-09T02:17:50,012 INFO [Time-limited test {}] server.Server(415): Started @3971ms 2024-11-09T02:17:50,507 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:17:50,515 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:17:50,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:17:50,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:17:50,518 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T02:17:50,519 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:17:50,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:17:50,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@183020ad{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/java.io.tmpdir/jetty-localhost-42555-hadoop-hdfs-3_4_1-tests_jar-_-any-17171914725236711445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:17:50,641 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:42555} 2024-11-09T02:17:50,641 INFO [Time-limited test {}] server.Server(415): Started @4600ms 2024-11-09T02:17:50,700 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T02:17:51,158 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/dfs/data/data2/current/BP-1620340638-172.17.0.2-1731118669061/current, will proceed with Du for space computation calculation, 2024-11-09T02:17:51,158 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/dfs/data/data1/current/BP-1620340638-172.17.0.2-1731118669061/current, will proceed with Du for space computation calculation, 2024-11-09T02:17:51,194 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T02:17:51,257 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc07ce0349d93cbe6 with lease ID 0xe149306c51564b61: Processing first storage report for DS-011365ec-6838-49e1-807b-95149c8d4ea8 from datanode DatanodeRegistration(127.0.0.1:32859, datanodeUuid=62356cfe-2e7b-498f-89e5-84d9dbecc6b0, infoPort=35811, infoSecurePort=0, ipcPort=36651, storageInfo=lv=-57;cid=testClusterID;nsid=484398714;c=1731118669061) 2024-11-09T02:17:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc07ce0349d93cbe6 with lease ID 0xe149306c51564b61: from storage DS-011365ec-6838-49e1-807b-95149c8d4ea8 node DatanodeRegistration(127.0.0.1:32859, datanodeUuid=62356cfe-2e7b-498f-89e5-84d9dbecc6b0, infoPort=35811, infoSecurePort=0, ipcPort=36651, storageInfo=lv=-57;cid=testClusterID;nsid=484398714;c=1731118669061), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T02:17:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc07ce0349d93cbe6 with lease ID 0xe149306c51564b61: Processing first storage report for DS-14c33212-c98b-4554-a783-6bd2352fe931 from datanode DatanodeRegistration(127.0.0.1:32859, datanodeUuid=62356cfe-2e7b-498f-89e5-84d9dbecc6b0, infoPort=35811, infoSecurePort=0, ipcPort=36651, storageInfo=lv=-57;cid=testClusterID;nsid=484398714;c=1731118669061) 2024-11-09T02:17:51,258 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc07ce0349d93cbe6 with lease ID 0xe149306c51564b61: from storage DS-14c33212-c98b-4554-a783-6bd2352fe931 node DatanodeRegistration(127.0.0.1:32859, datanodeUuid=62356cfe-2e7b-498f-89e5-84d9dbecc6b0, infoPort=35811, infoSecurePort=0, ipcPort=36651, storageInfo=lv=-57;cid=testClusterID;nsid=484398714;c=1731118669061), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T02:17:51,270 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023 2024-11-09T02:17:51,349 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/zookeeper_0, clientPort=57725, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T02:17:51,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57725 2024-11-09T02:17:51,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:51,374 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:51,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741825_1001 (size=7) 2024-11-09T02:17:51,999 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 with version=8 2024-11-09T02:17:52,000 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/hbase-staging 2024-11-09T02:17:52,121 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T02:17:52,382 INFO [Time-limited test {}] client.ConnectionUtils(129): master/54f3333e9615:0 server-side Connection retries=9 2024-11-09T02:17:52,402 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,402 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,402 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:17:52,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:17:52,532 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:17:52,590 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T02:17:52,599 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T02:17:52,602 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:17:52,629 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 13352 (auto-detected) 2024-11-09T02:17:52,630 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T02:17:52,648 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40839 2024-11-09T02:17:52,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:52,658 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:52,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40839 connecting to ZooKeeper ensemble=127.0.0.1:57725 2024-11-09T02:17:52,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408390x0, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:17:52,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40839-0x1018ad21b490000 connected 2024-11-09T02:17:52,731 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:17:52,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:17:52,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:17:52,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40839 2024-11-09T02:17:52,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40839 2024-11-09T02:17:52,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40839 2024-11-09T02:17:52,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40839 2024-11-09T02:17:52,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40839 2024-11-09T02:17:52,754 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277, hbase.cluster.distributed=false 2024-11-09T02:17:52,828 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/54f3333e9615:0 server-side Connection retries=9 2024-11-09T02:17:52,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,829 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:17:52,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:17:52,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:17:52,832 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:17:52,834 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:17:52,835 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45997 2024-11-09T02:17:52,837 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T02:17:52,842 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T02:17:52,844 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:52,847 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:52,851 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45997 connecting to ZooKeeper ensemble=127.0.0.1:57725 2024-11-09T02:17:52,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459970x0, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:17:52,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:459970x0, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:17:52,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45997-0x1018ad21b490001 connected 2024-11-09T02:17:52,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:17:52,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:17:52,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45997 2024-11-09T02:17:52,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45997 2024-11-09T02:17:52,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45997 2024-11-09T02:17:52,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45997 2024-11-09T02:17:52,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45997 2024-11-09T02:17:52,866 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/54f3333e9615,40839,1731118672115 2024-11-09T02:17:52,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:17:52,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:17:52,874 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/54f3333e9615,40839,1731118672115 2024-11-09T02:17:52,882 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;54f3333e9615:40839 2024-11-09T02:17:52,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:17:52,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:17:52,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:52,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:52,895 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T02:17:52,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T02:17:52,896 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/54f3333e9615,40839,1731118672115 from backup master directory 2024-11-09T02:17:52,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/54f3333e9615,40839,1731118672115 2024-11-09T02:17:52,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:17:52,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:17:52,899 WARN [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:17:52,899 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=54f3333e9615,40839,1731118672115 2024-11-09T02:17:52,901 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T02:17:52,903 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T02:17:52,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741826_1002 (size=42) 2024-11-09T02:17:53,377 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/hbase.id with ID: f828a386-725c-4ccb-86ad-3852885fd553 2024-11-09T02:17:53,419 INFO [master/54f3333e9615:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:17:53,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:53,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:53,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741827_1003 (size=196) 2024-11-09T02:17:53,877 INFO [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:17:53,879 INFO [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T02:17:53,897 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:17:53,902 INFO [master/54f3333e9615:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:17:53,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741828_1004 (size=1189) 2024-11-09T02:17:53,950 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store 2024-11-09T02:17:53,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741829_1005 (size=34) 2024-11-09T02:17:53,968 INFO [master/54f3333e9615:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T02:17:53,968 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:17:53,969 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:17:53,970 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:17:53,970 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:17:53,970 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:17:53,970 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:17:53,970 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:17:53,970 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T02:17:53,972 WARN [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/.initializing 2024-11-09T02:17:53,972 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/WALs/54f3333e9615,40839,1731118672115 2024-11-09T02:17:53,979 INFO [master/54f3333e9615:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:17:53,990 INFO [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C40839%2C1731118672115, suffix=, logDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/WALs/54f3333e9615,40839,1731118672115, archiveDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/oldWALs, maxLogs=10 2024-11-09T02:17:54,012 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/WALs/54f3333e9615,40839,1731118672115/54f3333e9615%2C40839%2C1731118672115.1731118673995, exclude list is [], retry=0 2024-11-09T02:17:54,028 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32859,DS-011365ec-6838-49e1-807b-95149c8d4ea8,DISK] 2024-11-09T02:17:54,031 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T02:17:54,067 INFO [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/WALs/54f3333e9615,40839,1731118672115/54f3333e9615%2C40839%2C1731118672115.1731118673995 2024-11-09T02:17:54,068 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35811:35811)] 2024-11-09T02:17:54,069 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:17:54,069 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:17:54,074 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,075 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T02:17:54,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:17:54,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T02:17:54,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:17:54,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T02:17:54,158 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:17:54,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T02:17:54,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:17:54,166 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,167 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,176 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:17:54,179 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:17:54,183 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:17:54,184 INFO [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59500783, jitterRate=-0.11336924135684967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:17:54,188 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T02:17:54,189 INFO [master/54f3333e9615:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T02:17:54,217 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1855159c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:17:54,251 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-09T02:17:54,262 INFO [master/54f3333e9615:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T02:17:54,263 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T02:17:54,265 INFO [master/54f3333e9615:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:17:54,266 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T02:17:54,271 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:17:54,271 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T02:17:54,296 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T02:17:54,306 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T02:17:54,308 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-09T02:17:54,310 INFO [master/54f3333e9615:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T02:17:54,311 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T02:17:54,313 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-09T02:17:54,315 INFO [master/54f3333e9615:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T02:17:54,317 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T02:17:54,319 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-09T02:17:54,320 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T02:17:54,321 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T02:17:54,330 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T02:17:54,331 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T02:17:54,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:17:54,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:17:54,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,335 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=54f3333e9615,40839,1731118672115, sessionid=0x1018ad21b490000, setting cluster-up flag (Was=false) 2024-11-09T02:17:54,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,353 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T02:17:54,354 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54f3333e9615,40839,1731118672115 2024-11-09T02:17:54,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:17:54,366 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T02:17:54,368 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54f3333e9615,40839,1731118672115 2024-11-09T02:17:54,457 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-09T02:17:54,465 INFO [master/54f3333e9615:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-09T02:17:54,467 INFO [master/54f3333e9615:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T02:17:54,474 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 54f3333e9615,40839,1731118672115 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T02:17:54,477 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/54f3333e9615:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/54f3333e9615:0, corePoolSize=10, maxPoolSize=10 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,478 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/54f3333e9615:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:17:54,479 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,480 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118704480 2024-11-09T02:17:54,482 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T02:17:54,482 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;54f3333e9615:45997 2024-11-09T02:17:54,483 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T02:17:54,484 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1008): ClusterId : f828a386-725c-4ccb-86ad-3852885fd553 2024-11-09T02:17:54,485 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:17:54,485 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-09T02:17:54,487 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T02:17:54,488 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T02:17:54,488 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T02:17:54,488 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T02:17:54,488 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T02:17:54,492 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,492 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T02:17:54,494 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,498 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T02:17:54,498 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T02:17:54,499 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T02:17:54,500 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T02:17:54,500 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T02:17:54,502 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T02:17:54,503 DEBUG [RS:0;54f3333e9615:45997 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41780640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:17:54,503 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T02:17:54,504 INFO [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T02:17:54,508 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.large.0-1731118674505,5,FailOnTimeoutGroup] 2024-11-09T02:17:54,508 DEBUG [RS:0;54f3333e9615:45997 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbd83ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54f3333e9615/172.17.0.2:0 2024-11-09T02:17:54,509 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.small.0-1731118674508,5,FailOnTimeoutGroup] 2024-11-09T02:17:54,509 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,509 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T02:17:54,511 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741831_1007 (size=1039) 2024-11-09T02:17:54,511 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,512 INFO [RS:0;54f3333e9615:45997 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-09T02:17:54,512 INFO [RS:0;54f3333e9615:45997 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-09T02:17:54,513 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-09T02:17:54,513 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-09T02:17:54,513 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:17:54,515 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3073): reportForDuty to master=54f3333e9615,40839,1731118672115 with isa=54f3333e9615/172.17.0.2:45997, startcode=1731118672827 2024-11-09T02:17:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741832_1008 (size=32) 2024-11-09T02:17:54,530 DEBUG [RS:0;54f3333e9615:45997 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:17:54,572 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44249, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:17:54,580 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40839 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,582 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40839 {}] master.ServerManager(486): Registering regionserver=54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,602 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:17:54,602 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:40811 2024-11-09T02:17:54,602 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-09T02:17:54,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:17:54,609 DEBUG [RS:0;54f3333e9615:45997 {}] zookeeper.ZKUtil(111): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,609 WARN [RS:0;54f3333e9615:45997 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:17:54,609 INFO [RS:0;54f3333e9615:45997 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:17:54,610 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,611 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54f3333e9615,45997,1731118672827] 2024-11-09T02:17:54,630 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-09T02:17:54,643 INFO [RS:0;54f3333e9615:45997 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T02:17:54,658 INFO [RS:0;54f3333e9615:45997 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T02:17:54,661 INFO [RS:0;54f3333e9615:45997 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T02:17:54,661 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,663 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-09T02:17:54,672 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,673 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,673 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,673 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,674 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,674 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,674 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54f3333e9615:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:17:54,674 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,675 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,675 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,675 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,676 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54f3333e9615:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:17:54,676 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54f3333e9615:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:17:54,676 DEBUG [RS:0;54f3333e9615:45997 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54f3333e9615:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:17:54,680 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,680 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,680 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,680 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,681 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,45997,1731118672827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:17:54,710 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T02:17:54,712 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,45997,1731118672827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:17:54,732 INFO [RS:0;54f3333e9615:45997 {}] regionserver.Replication(204): 54f3333e9615,45997,1731118672827 started 2024-11-09T02:17:54,732 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1767): Serving as 54f3333e9615,45997,1731118672827, RpcServer on 54f3333e9615/172.17.0.2:45997, sessionid=0x1018ad21b490001 2024-11-09T02:17:54,733 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T02:17:54,733 DEBUG [RS:0;54f3333e9615:45997 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,733 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54f3333e9615,45997,1731118672827' 2024-11-09T02:17:54,733 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T02:17:54,734 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T02:17:54,735 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T02:17:54,735 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T02:17:54,735 DEBUG [RS:0;54f3333e9615:45997 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54f3333e9615,45997,1731118672827 2024-11-09T02:17:54,735 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54f3333e9615,45997,1731118672827' 2024-11-09T02:17:54,735 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T02:17:54,736 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T02:17:54,736 DEBUG [RS:0;54f3333e9615:45997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T02:17:54,737 INFO [RS:0;54f3333e9615:45997 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T02:17:54,737 INFO [RS:0;54f3333e9615:45997 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T02:17:54,932 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:17:54,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:17:54,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:17:54,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:17:54,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:17:54,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:17:54,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:17:54,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:17:54,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:17:54,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:17:54,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:17:54,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740 2024-11-09T02:17:54,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740 2024-11-09T02:17:54,950 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-09T02:17:54,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T02:17:54,956 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:17:54,957 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67431410, jitterRate=0.004806309938430786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-09T02:17:54,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T02:17:54,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:17:54,959 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T02:17:54,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T02:17:54,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:17:54,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:17:54,961 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T02:17:54,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T02:17:54,963 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:17:54,963 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-09T02:17:54,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T02:17:54,977 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:17:54,979 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T02:17:55,130 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:17:56,131 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:17:57,131 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:17:58,132 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:17:59,133 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:00,134 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:01,134 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:02,135 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:02,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T02:18:02,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:03,137 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:04,138 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:05,139 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:06,140 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:07,140 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:08,141 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:09,142 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:10,143 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:11,144 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:12,145 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:13,146 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:14,146 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:15,147 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:16,148 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:17,149 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:18,149 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:19,150 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:20,151 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:21,151 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:21,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:18:21,452 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:18:22,152 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:23,153 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:24,153 WARN [54f3333e9615:40839 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:18:24,507 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:18:24,511 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:18:24,742 INFO [RS:0;54f3333e9615:45997 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:18:24,747 INFO [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C45997%2C1731118672827, suffix=, logDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827, archiveDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/oldWALs, maxLogs=32 2024-11-09T02:18:24,764 DEBUG [RS:0;54f3333e9615:45997 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827/54f3333e9615%2C45997%2C1731118672827.1731118704749, exclude list is [], retry=0 2024-11-09T02:18:24,769 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32859,DS-011365ec-6838-49e1-807b-95149c8d4ea8,DISK] 2024-11-09T02:18:24,773 INFO [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827/54f3333e9615%2C45997%2C1731118672827.1731118704749 2024-11-09T02:18:24,773 DEBUG [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35811:35811)] 2024-11-09T02:18:24,905 DEBUG [54f3333e9615:40839 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T02:18:24,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:24,915 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54f3333e9615,45997,1731118672827, state=OPENING 2024-11-09T02:18:24,921 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T02:18:24,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:18:24,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:18:24,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:18:24,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:18:24,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:25,099 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:25,101 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T02:18:25,103 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35810, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T02:18:25,115 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-09T02:18:25,115 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:18:25,116 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T02:18:25,119 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54f3333e9615%2C45997%2C1731118672827.meta, suffix=.meta, logDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827, archiveDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/oldWALs, maxLogs=32 2024-11-09T02:18:25,137 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827/54f3333e9615%2C45997%2C1731118672827.meta.1731118705122.meta, exclude list is [], retry=0 2024-11-09T02:18:25,141 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32859,DS-011365ec-6838-49e1-807b-95149c8d4ea8,DISK] 2024-11-09T02:18:25,144 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827/54f3333e9615%2C45997%2C1731118672827.meta.1731118705122.meta 2024-11-09T02:18:25,145 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35811:35811)] 2024-11-09T02:18:25,145 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:25,146 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T02:18:25,210 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T02:18:25,215 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T02:18:25,219 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T02:18:25,219 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:25,219 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-09T02:18:25,220 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-09T02:18:25,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:18:25,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:18:25,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:25,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:18:25,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:18:25,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:18:25,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:25,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:18:25,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:18:25,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:18:25,232 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:25,232 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:18:25,234 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740 2024-11-09T02:18:25,237 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740 2024-11-09T02:18:25,239 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-09T02:18:25,241 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T02:18:25,243 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60732480, jitterRate=-0.0950155258178711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-09T02:18:25,245 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T02:18:25,252 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731118705094 2024-11-09T02:18:25,263 DEBUG [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T02:18:25,264 INFO [RS_OPEN_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-09T02:18:25,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:25,266 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54f3333e9615,45997,1731118672827, state=OPEN 2024-11-09T02:18:25,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:18:25,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:18:25,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:18:25,273 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:18:25,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T02:18:25,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=54f3333e9615,45997,1731118672827 in 348 msec 2024-11-09T02:18:25,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T02:18:25,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3090 sec 2024-11-09T02:18:25,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 30.8850 sec 2024-11-09T02:18:25,289 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731118705288, completionTime=-1 2024-11-09T02:18:25,289 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T02:18:25,289 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-09T02:18:25,325 DEBUG [hconnection-0x6681570d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:25,327 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35820, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:25,336 INFO [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-09T02:18:25,336 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118765336 2024-11-09T02:18:25,336 INFO [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118825336 2024-11-09T02:18:25,337 INFO [master/54f3333e9615:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 47 msec 2024-11-09T02:18:25,356 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:25,356 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:25,357 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:25,358 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-54f3333e9615:40839, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:25,358 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:25,363 DEBUG [master/54f3333e9615:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-09T02:18:25,366 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-09T02:18:25,367 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T02:18:25,373 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-09T02:18:25,376 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:25,377 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:25,379 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:25,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741835_1011 (size=358) 2024-11-09T02:18:25,810 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 6235bfc49272e5114ec5c10328305fa2, NAME => 'hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:25,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741836_1012 (size=42) 2024-11-09T02:18:25,872 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:25,872 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 6235bfc49272e5114ec5c10328305fa2, disabling compactions & flushes 2024-11-09T02:18:25,872 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:25,873 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:25,873 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. after waiting 0 ms 2024-11-09T02:18:25,874 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:25,874 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:25,874 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 6235bfc49272e5114ec5c10328305fa2: 2024-11-09T02:18:25,877 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:25,888 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731118705881"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118705881"}]},"ts":"1731118705881"} 2024-11-09T02:18:25,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:25,921 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:25,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118705921"}]},"ts":"1731118705921"} 2024-11-09T02:18:25,932 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-09T02:18:25,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6235bfc49272e5114ec5c10328305fa2, ASSIGN}] 2024-11-09T02:18:25,942 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=6235bfc49272e5114ec5c10328305fa2, ASSIGN 2024-11-09T02:18:25,945 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=6235bfc49272e5114ec5c10328305fa2, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:26,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6235bfc49272e5114ec5c10328305fa2, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:26,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 6235bfc49272e5114ec5c10328305fa2, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:26,260 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:26,268 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:26,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 6235bfc49272e5114ec5c10328305fa2, NAME => 'hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:26,269 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,269 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:26,269 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,272 INFO [StoreOpener-6235bfc49272e5114ec5c10328305fa2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,275 INFO [StoreOpener-6235bfc49272e5114ec5c10328305fa2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6235bfc49272e5114ec5c10328305fa2 columnFamilyName info 2024-11-09T02:18:26,276 DEBUG [StoreOpener-6235bfc49272e5114ec5c10328305fa2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:26,277 INFO [StoreOpener-6235bfc49272e5114ec5c10328305fa2-1 {}] regionserver.HStore(327): Store=6235bfc49272e5114ec5c10328305fa2/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:26,278 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,279 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,284 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:18:26,288 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:26,289 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 6235bfc49272e5114ec5c10328305fa2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63122558, jitterRate=-0.059400588274002075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:26,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 6235bfc49272e5114ec5c10328305fa2: 2024-11-09T02:18:26,294 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2., pid=6, masterSystemTime=1731118706260 2024-11-09T02:18:26,298 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:26,299 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:18:26,300 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=6235bfc49272e5114ec5c10328305fa2, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:26,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T02:18:26,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 6235bfc49272e5114ec5c10328305fa2, server=54f3333e9615,45997,1731118672827 in 200 msec 2024-11-09T02:18:26,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T02:18:26,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=6235bfc49272e5114ec5c10328305fa2, ASSIGN in 371 msec 2024-11-09T02:18:26,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:26,317 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118706317"}]},"ts":"1731118706317"} 2024-11-09T02:18:26,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-09T02:18:26,325 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:26,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 959 msec 2024-11-09T02:18:26,378 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-09T02:18:26,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-09T02:18:26,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:18:26,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:18:26,418 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-09T02:18:26,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T02:18:26,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec 2024-11-09T02:18:26,453 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-09T02:18:26,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T02:18:26,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 21 msec 2024-11-09T02:18:26,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-09T02:18:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-09T02:18:26,508 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 33.609sec 2024-11-09T02:18:26,510 INFO [master/54f3333e9615:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T02:18:26,512 INFO [master/54f3333e9615:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T02:18:26,513 INFO [master/54f3333e9615:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T02:18:26,514 INFO [master/54f3333e9615:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T02:18:26,514 INFO [master/54f3333e9615:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T02:18:26,515 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:18:26,516 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T02:18:26,529 DEBUG [master/54f3333e9615:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-09T02:18:26,530 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T02:18:26,530 INFO [master/54f3333e9615:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54f3333e9615,40839,1731118672115-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:18:26,615 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x574a453f to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61e4a9fc 2024-11-09T02:18:26,616 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-09T02:18:26,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d933e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:26,639 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T02:18:26,639 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T02:18:26,651 DEBUG [hconnection-0x35085285-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:26,678 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35834, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:26,692 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=54f3333e9615,40839,1731118672115 2024-11-09T02:18:26,715 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=10208 2024-11-09T02:18:26,721 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x312a75fa to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42bb6992 2024-11-09T02:18:26,730 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x312a75fa {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd647b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:26,735 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x144b7569 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d25698b 2024-11-09T02:18:26,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ae8247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:26,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:18:26,764 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52576, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:18:26,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-09T02:18:26,782 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:26,782 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:26,785 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:26,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_0_-false" procId is: 9 2024-11-09T02:18:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T02:18:26,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741837_1013 (size=437) 2024-11-09T02:18:27,226 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c7cadccf0db1443c03afbf4a8ffb7c74, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:27,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741838_1014 (size=88) 2024-11-09T02:18:27,238 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:27,238 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing c7cadccf0db1443c03afbf4a8ffb7c74, disabling compactions & flushes 2024-11-09T02:18:27,238 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,238 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,238 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. after waiting 0 ms 2024-11-09T02:18:27,239 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,239 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,239 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for c7cadccf0db1443c03afbf4a8ffb7c74: 2024-11-09T02:18:27,242 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:27,242 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731118707242"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118707242"}]},"ts":"1731118707242"} 2024-11-09T02:18:27,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:27,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:27,250 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118707250"}]},"ts":"1731118707250"} 2024-11-09T02:18:27,253 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:27,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=c7cadccf0db1443c03afbf4a8ffb7c74, ASSIGN}] 2024-11-09T02:18:27,262 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=c7cadccf0db1443c03afbf4a8ffb7c74, ASSIGN 2024-11-09T02:18:27,264 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=c7cadccf0db1443c03afbf4a8ffb7c74, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:27,415 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c7cadccf0db1443c03afbf4a8ffb7c74, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:27,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure c7cadccf0db1443c03afbf4a8ffb7c74, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:27,575 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:27,582 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,583 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => c7cadccf0db1443c03afbf4a8ffb7c74, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:27,583 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,583 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:27,584 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,584 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,587 INFO [StoreOpener-c7cadccf0db1443c03afbf4a8ffb7c74-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,590 INFO [StoreOpener-c7cadccf0db1443c03afbf4a8ffb7c74-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7cadccf0db1443c03afbf4a8ffb7c74 columnFamilyName testFamily 2024-11-09T02:18:27,590 DEBUG [StoreOpener-c7cadccf0db1443c03afbf4a8ffb7c74-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:27,591 INFO [StoreOpener-c7cadccf0db1443c03afbf4a8ffb7c74-1 {}] regionserver.HStore(327): Store=c7cadccf0db1443c03afbf4a8ffb7c74/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:27,592 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,593 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,597 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:18:27,603 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:27,604 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened c7cadccf0db1443c03afbf4a8ffb7c74; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65578636, jitterRate=-0.02280217409133911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:27,606 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for c7cadccf0db1443c03afbf4a8ffb7c74: 2024-11-09T02:18:27,608 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74., pid=11, masterSystemTime=1731118707575 2024-11-09T02:18:27,613 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,613 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:18:27,613 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c7cadccf0db1443c03afbf4a8ffb7c74, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:27,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-09T02:18:27,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure c7cadccf0db1443c03afbf4a8ffb7c74, server=54f3333e9615,45997,1731118672827 in 197 msec 2024-11-09T02:18:27,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T02:18:27,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, region=c7cadccf0db1443c03afbf4a8ffb7c74, ASSIGN in 383 msec 2024-11-09T02:18:27,650 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:27,650 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118707650"}]},"ts":"1731118707650"} 2024-11-09T02:18:27,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:27,663 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:27,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_0_-false in 886 msec 2024-11-09T02:18:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T02:18:27,805 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_0_-false, procId: 9 completed 2024-11-09T02:18:27,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:27,807 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:27,812 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:27,812 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:27,812 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_0_-false assigned. 2024-11-09T02:18:27,816 DEBUG [hconnection-0x6b471694-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:27,818 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35850, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:27,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:27,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T02:18:27,856 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:18:27,858 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x312a75fa {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:18:27,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:27,865 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35860, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:18:27,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T02:18:27,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269707 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T02:18:27,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269707 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281909 2024-11-09T02:18:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35860 deadline: 1731118710878, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:27,910 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269707 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:18:27,925 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T02:18:27,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269708 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T02:18:27,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:27,931 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:27,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269709 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281907 2024-11-09T02:18:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35860 deadline: 1731118710930, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:27,935 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269709 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:18:27,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:27,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269710 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1481684882995281906 2024-11-09T02:18:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35860 deadline: 1731118710936, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:27,939 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269710 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:18:27,944 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:27,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269711 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:27,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1481684882995281905 2024-11-09T02:18:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35860 deadline: 1731118710944, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:27,947 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269711 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:18:27,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:27,950 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x144b7569 to 127.0.0.1:57725 2024-11-09T02:18:27,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:27,951 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:27,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:27,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:27,958 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x312a75fa to 127.0.0.1:57725 2024-11-09T02:18:27,972 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[0] Thread=232 (was 220) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x144b7569 java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Thread.join(Thread.java:1313) java.base@17.0.11/java.lang.Thread.join(Thread.java:1381) app//org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1523) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1547) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f56b0b458e0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x312a75fa-SendThread(127.0.0.1:57725) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: hconnection-0x35085285-shared-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x312a75fa-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x144b7569-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/54f3333e9615:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x144b7569-SendThread(127.0.0.1:57725) java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1395) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_177180086_22 at /127.0.0.1:60770 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57725@0x312a75fa java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) app//org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1543) app//org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:348) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:398) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$738/0x00007f56b0b458e0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 399), ProcessCount=11 (was 11), AvailableMemoryMB=10154 (was 10208) 2024-11-09T02:18:27,983 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=232, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=10154 2024-11-09T02:18:27,985 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e567203 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@488bbf84 2024-11-09T02:18:27,989 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x3e567203 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6326ac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:27,991 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b4e4ad2 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1760f201 2024-11-09T02:18:27,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1648742b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:28,005 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:28,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:18:28,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:28,009 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:28,009 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_0_-false" procId is: 12 2024-11-09T02:18:28,011 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T02:18:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741839_1015 (size=423) 2024-11-09T02:18:28,042 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 27cb43c1981e47632a87e1850b149c1e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:28,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741840_1016 (size=74) 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 27cb43c1981e47632a87e1850b149c1e, disabling compactions & flushes 2024-11-09T02:18:28,459 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. after waiting 0 ms 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,459 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,459 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 27cb43c1981e47632a87e1850b149c1e: 2024-11-09T02:18:28,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:28,462 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731118708462"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118708462"}]},"ts":"1731118708462"} 2024-11-09T02:18:28,467 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:28,469 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:28,470 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118708469"}]},"ts":"1731118708469"} 2024-11-09T02:18:28,473 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:28,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=27cb43c1981e47632a87e1850b149c1e, ASSIGN}] 2024-11-09T02:18:28,480 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=27cb43c1981e47632a87e1850b149c1e, ASSIGN 2024-11-09T02:18:28,482 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=27cb43c1981e47632a87e1850b149c1e, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:28,634 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=27cb43c1981e47632a87e1850b149c1e, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:28,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; OpenRegionProcedure 27cb43c1981e47632a87e1850b149c1e, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:28,793 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:28,800 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7285): Opening region: {ENCODED => 27cb43c1981e47632a87e1850b149c1e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:28,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_0_-false 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:28,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7327): checking encryption for 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(7330): checking classloading for 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,804 INFO [StoreOpener-27cb43c1981e47632a87e1850b149c1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,807 INFO [StoreOpener-27cb43c1981e47632a87e1850b149c1e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27cb43c1981e47632a87e1850b149c1e columnFamilyName testFamily 2024-11-09T02:18:28,807 DEBUG [StoreOpener-27cb43c1981e47632a87e1850b149c1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:28,808 INFO [StoreOpener-27cb43c1981e47632a87e1850b149c1e-1 {}] regionserver.HStore(327): Store=27cb43c1981e47632a87e1850b149c1e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:28,811 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,817 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1085): writing seq id for 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:18:28,821 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:28,822 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1102): Opened 27cb43c1981e47632a87e1850b149c1e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73721188, jitterRate=0.09853130578994751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:28,824 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegion(1001): Region open journal for 27cb43c1981e47632a87e1850b149c1e: 2024-11-09T02:18:28,826 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e., pid=14, masterSystemTime=1731118708793 2024-11-09T02:18:28,830 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,830 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=14}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:18:28,831 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=27cb43c1981e47632a87e1850b149c1e, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:28,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T02:18:28,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; OpenRegionProcedure 27cb43c1981e47632a87e1850b149c1e, server=54f3333e9615,45997,1731118672827 in 197 msec 2024-11-09T02:18:28,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-09T02:18:28,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, region=27cb43c1981e47632a87e1850b149c1e, ASSIGN in 362 msec 2024-11-09T02:18:28,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:28,855 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118708855"}]},"ts":"1731118708855"} 2024-11-09T02:18:28,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:28,865 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=12, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:28,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_0_-false in 860 msec 2024-11-09T02:18:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T02:18:29,021 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_0_-false, procId: 12 completed 2024-11-09T02:18:29,021 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:29,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:29,028 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:29,028 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:29,028 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-09T02:18:29,032 DEBUG [hconnection-0x512e1248-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:29,034 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35874, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:29,057 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:29,064 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:18:29,074 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:18:30,761 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:18:30,834 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' 2024-11-09T02:18:30,836 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_0_-false' 2024-11-09T02:18:30,837 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T02:18:30,839 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-09T02:18:32,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-09T02:18:32,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:32,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T02:18:32,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:32,590 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-09T02:18:32,590 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:32,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:18:32,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T02:18:32,592 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:18:32,592 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:35,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T02:18:36,096 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:18:38,076 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:18:38,077 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T02:18:38.076Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11013: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_0_-false' at region=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1980ms, rpcTimeout=1977ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:18:38,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9013 2024-11-09T02:18:38,078 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b4e4ad2 to 127.0.0.1:57725 2024-11-09T02:18:38,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:38,078 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:38,079 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:38,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:38,079 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e567203 to 127.0.0.1:57725 2024-11-09T02:18:38,090 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[0] Thread=225 (was 232), OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 399) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10123 (was 10154) 2024-11-09T02:18:38,099 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=225, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=10123 2024-11-09T02:18:38,102 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b8ee3d4 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51fb2da6 2024-11-09T02:18:38,106 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2b8ee3d4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b53fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:38,108 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c55ebef to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e7c030c 2024-11-09T02:18:38,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2230cf54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:38,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:38,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false 2024-11-09T02:18:38,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:38,120 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:38,120 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_0_-false" procId is: 15 2024-11-09T02:18:38,122 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T02:18:38,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741841_1017 (size=421) 2024-11-09T02:18:38,536 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 970ccc55dc0014f96f14429b0893f112, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:38,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741842_1018 (size=72) 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 970ccc55dc0014f96f14429b0893f112, disabling compactions & flushes 2024-11-09T02:18:38,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. after waiting 0 ms 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:38,951 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:38,951 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 970ccc55dc0014f96f14429b0893f112: 2024-11-09T02:18:38,953 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:38,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731118718953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118718953"}]},"ts":"1731118718953"} 2024-11-09T02:18:38,957 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:38,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:38,959 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118718959"}]},"ts":"1731118718959"} 2024-11-09T02:18:38,962 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:38,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=970ccc55dc0014f96f14429b0893f112, ASSIGN}] 2024-11-09T02:18:38,970 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=970ccc55dc0014f96f14429b0893f112, ASSIGN 2024-11-09T02:18:38,971 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=15, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=970ccc55dc0014f96f14429b0893f112, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:39,122 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=970ccc55dc0014f96f14429b0893f112, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T02:18:39,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; OpenRegionProcedure 970ccc55dc0014f96f14429b0893f112, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:39,281 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:39,289 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:39,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => 970ccc55dc0014f96f14429b0893f112, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:39,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_0_-false 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:39,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,293 INFO [StoreOpener-970ccc55dc0014f96f14429b0893f112-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,295 INFO [StoreOpener-970ccc55dc0014f96f14429b0893f112-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 970ccc55dc0014f96f14429b0893f112 columnFamilyName testFamily 2024-11-09T02:18:39,296 DEBUG [StoreOpener-970ccc55dc0014f96f14429b0893f112-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:39,297 INFO [StoreOpener-970ccc55dc0014f96f14429b0893f112-1 {}] regionserver.HStore(327): Store=970ccc55dc0014f96f14429b0893f112/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:39,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:18:39,312 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:39,313 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1102): Opened 970ccc55dc0014f96f14429b0893f112; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69696463, jitterRate=0.038558229804039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:39,314 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for 970ccc55dc0014f96f14429b0893f112: 2024-11-09T02:18:39,316 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112., pid=17, masterSystemTime=1731118719281 2024-11-09T02:18:39,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:39,320 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=17}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:18:39,321 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=970ccc55dc0014f96f14429b0893f112, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:39,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-09T02:18:39,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; OpenRegionProcedure 970ccc55dc0014f96f14429b0893f112, server=54f3333e9615,45997,1731118672827 in 197 msec 2024-11-09T02:18:39,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T02:18:39,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false, region=970ccc55dc0014f96f14429b0893f112, ASSIGN in 360 msec 2024-11-09T02:18:39,332 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:39,332 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118719332"}]},"ts":"1731118719332"} 2024-11-09T02:18:39,335 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:39,339 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=15, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:39,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_0_-false in 1.2230 sec 2024-11-09T02:18:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-09T02:18:41,127 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_0_-false, procId: 15 completed 2024-11-09T02:18:41,128 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:41,128 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:41,132 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:41,133 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:41,133 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_0_-false assigned. 2024-11-09T02:18:41,135 DEBUG [hconnection-0x6343ed0b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:41,138 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38274, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:41,152 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:41,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:18:41,156 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:18:41,158 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2b8ee3d4 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:18:41,158 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2b8ee3d4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:41,160 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38284, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:18:41,164 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:18:41,164 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269723 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:41,165 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:18:41,166 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269723 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:41,167 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T02:18:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-09T02:18:42,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:42,597 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:18:42,841 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_0_-false' 2024-11-09T02:18:44,844 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:18:50,175 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9009 2024-11-09T02:18:50,175 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c55ebef to 127.0.0.1:57725 2024-11-09T02:18:50,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:50,175 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269723 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:18:50,176 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:50,176 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:50,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281893' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T02:18:50,176 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b8ee3d4 to 127.0.0.1:57725 2024-11-09T02:18:50,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:38284 deadline: 1731118736175, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281893' does not exist 2024-11-09T02:18:50,179 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:18:50,176 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 54f3333e9615,45997,1731118672827 for closing scanner id = -1481684882995281893 for 970ccc55dc0014f96f14429b0893f112 of hbase:testMetaScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T02:18:50,186 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[0] Thread=224 (was 225), OpenFileDescriptor=448 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 422), ProcessCount=11 (was 11), AvailableMemoryMB=10116 (was 10123) 2024-11-09T02:18:50,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=224, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=10116 2024-11-09T02:18:50,196 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75859c3c to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b6678da 2024-11-09T02:18:50,199 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x75859c3c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2aad80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:50,201 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58325c54 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d914229 2024-11-09T02:18:50,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142c43db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:50,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false 2024-11-09T02:18:50,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:50,210 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:50,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_0_-false" procId is: 18 2024-11-09T02:18:50,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T02:18:50,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741843_1019 (size=432) 2024-11-09T02:18:50,624 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ac58767bfb05dc96875b60142d320308, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:50,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741844_1020 (size=83) 2024-11-09T02:18:50,635 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:50,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ac58767bfb05dc96875b60142d320308, disabling compactions & flushes 2024-11-09T02:18:50,636 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. after waiting 0 ms 2024-11-09T02:18:50,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,636 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,636 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ac58767bfb05dc96875b60142d320308: 2024-11-09T02:18:50,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:50,638 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731118730638"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118730638"}]},"ts":"1731118730638"} 2024-11-09T02:18:50,642 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:50,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:50,643 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118730643"}]},"ts":"1731118730643"} 2024-11-09T02:18:50,646 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:50,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=ac58767bfb05dc96875b60142d320308, ASSIGN}] 2024-11-09T02:18:50,658 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=ac58767bfb05dc96875b60142d320308, ASSIGN 2024-11-09T02:18:50,663 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=19, ppid=18, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=ac58767bfb05dc96875b60142d320308, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:50,668 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:18:50,814 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=ac58767bfb05dc96875b60142d320308, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:50,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; OpenRegionProcedure ac58767bfb05dc96875b60142d320308, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:50,970 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:50,977 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,977 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7285): Opening region: {ENCODED => ac58767bfb05dc96875b60142d320308, NAME => 'testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:50,977 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_0_-false ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,977 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:50,978 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7327): checking encryption for ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,978 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7330): checking classloading for ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,980 INFO [StoreOpener-ac58767bfb05dc96875b60142d320308-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,982 INFO [StoreOpener-ac58767bfb05dc96875b60142d320308-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac58767bfb05dc96875b60142d320308 columnFamilyName testFamily 2024-11-09T02:18:50,982 DEBUG [StoreOpener-ac58767bfb05dc96875b60142d320308-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:50,982 INFO [StoreOpener-ac58767bfb05dc96875b60142d320308-1 {}] regionserver.HStore(327): Store=ac58767bfb05dc96875b60142d320308/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:50,983 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,984 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,987 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1085): writing seq id for ac58767bfb05dc96875b60142d320308 2024-11-09T02:18:50,990 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:50,991 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1102): Opened ac58767bfb05dc96875b60142d320308; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71138498, jitterRate=0.060046225786209106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:50,991 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1001): Region open journal for ac58767bfb05dc96875b60142d320308: 2024-11-09T02:18:50,993 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308., pid=20, masterSystemTime=1731118730970 2024-11-09T02:18:50,995 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,995 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:18:50,996 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=19 updating hbase:meta row=ac58767bfb05dc96875b60142d320308, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:51,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T02:18:51,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; OpenRegionProcedure ac58767bfb05dc96875b60142d320308, server=54f3333e9615,45997,1731118672827 in 181 msec 2024-11-09T02:18:51,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-09T02:18:51,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_0_-false, region=ac58767bfb05dc96875b60142d320308, ASSIGN in 346 msec 2024-11-09T02:18:51,004 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:51,004 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118731004"}]},"ts":"1731118731004"} 2024-11-09T02:18:51,006 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:51,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=18, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:51,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_0_-false in 802 msec 2024-11-09T02:18:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T02:18:51,214 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_0_-false, procId: 18 completed 2024-11-09T02:18:51,214 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:51,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:51,219 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:51,219 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:51,219 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_0_-false assigned. 2024-11-09T02:18:51,222 DEBUG [hconnection-0x594e8579-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:51,224 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60918, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:51,235 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:51,238 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T02:18:51,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T02:18:51,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269727 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,242 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T02:18:51,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269727 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281889 2024-11-09T02:18:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60918 deadline: 1731118732242, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:51,244 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:18:51,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T02:18:51,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269728 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,247 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T02:18:51,247 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:51,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:51,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269729 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281887 2024-11-09T02:18:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60918 deadline: 1731118732249, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:51,250 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:18:51,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:51,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269730 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1481684882995281886 2024-11-09T02:18:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60918 deadline: 1731118732252, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:51,253 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:18:51,255 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:51,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269731 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:51,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1481684882995281885 2024-11-09T02:18:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60918 deadline: 1731118732255, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:18:51,257 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:18:51,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:18:51,259 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58325c54 to 127.0.0.1:57725 2024-11-09T02:18:51,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:51,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:51,259 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:51,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:51,259 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75859c3c to 127.0.0.1:57725 2024-11-09T02:18:51,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:18:51,269 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[0] Thread=226 (was 224) - Thread LEAK? -, OpenFileDescriptor=454 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=10068 (was 10116) 2024-11-09T02:18:51,277 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=226, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=10068 2024-11-09T02:18:51,278 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a9c4477 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@428a041b 2024-11-09T02:18:51,282 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x7a9c4477 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d0757d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:51,283 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d52f25d to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@50cc5602 2024-11-09T02:18:51,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486b0f90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:51,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false 2024-11-09T02:18:51,292 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:51,292 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:51,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_0_-false" procId is: 21 2024-11-09T02:18:51,293 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-09T02:18:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741845_1021 (size=426) 2024-11-09T02:18:51,304 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 04a6b35baf34def60d1b7d01a8823d80, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:51,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741846_1022 (size=77) 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 04a6b35baf34def60d1b7d01a8823d80, disabling compactions & flushes 2024-11-09T02:18:51,719 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. after waiting 0 ms 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:51,719 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:51,719 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 04a6b35baf34def60d1b7d01a8823d80: 2024-11-09T02:18:51,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:51,722 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731118731721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118731721"}]},"ts":"1731118731721"} 2024-11-09T02:18:51,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:51,725 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:51,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118731725"}]},"ts":"1731118731725"} 2024-11-09T02:18:51,727 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:51,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=04a6b35baf34def60d1b7d01a8823d80, ASSIGN}] 2024-11-09T02:18:51,733 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=04a6b35baf34def60d1b7d01a8823d80, ASSIGN 2024-11-09T02:18:51,734 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=04a6b35baf34def60d1b7d01a8823d80, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:51,884 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=04a6b35baf34def60d1b7d01a8823d80, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:51,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; OpenRegionProcedure 04a6b35baf34def60d1b7d01a8823d80, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:52,040 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:52,045 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:52,045 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7285): Opening region: {ENCODED => 04a6b35baf34def60d1b7d01a8823d80, NAME => 'testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:52,046 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_0_-false 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,046 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:52,046 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7327): checking encryption for 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,046 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7330): checking classloading for 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,048 INFO [StoreOpener-04a6b35baf34def60d1b7d01a8823d80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,051 INFO [StoreOpener-04a6b35baf34def60d1b7d01a8823d80-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04a6b35baf34def60d1b7d01a8823d80 columnFamilyName testFamily 2024-11-09T02:18:52,051 DEBUG [StoreOpener-04a6b35baf34def60d1b7d01a8823d80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:52,052 INFO [StoreOpener-04a6b35baf34def60d1b7d01a8823d80-1 {}] regionserver.HStore(327): Store=04a6b35baf34def60d1b7d01a8823d80/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:52,053 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,054 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,056 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1085): writing seq id for 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:18:52,059 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:52,060 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1102): Opened 04a6b35baf34def60d1b7d01a8823d80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73268187, jitterRate=0.09178106486797333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:52,060 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1001): Region open journal for 04a6b35baf34def60d1b7d01a8823d80: 2024-11-09T02:18:52,062 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80., pid=23, masterSystemTime=1731118732040 2024-11-09T02:18:52,064 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:52,064 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:18:52,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=22 updating hbase:meta row=04a6b35baf34def60d1b7d01a8823d80, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:52,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T02:18:52,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; OpenRegionProcedure 04a6b35baf34def60d1b7d01a8823d80, server=54f3333e9615,45997,1731118672827 in 180 msec 2024-11-09T02:18:52,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T02:18:52,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_0_-false, region=04a6b35baf34def60d1b7d01a8823d80, ASSIGN in 338 msec 2024-11-09T02:18:52,073 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:52,073 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118732073"}]},"ts":"1731118732073"} 2024-11-09T02:18:52,075 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:52,078 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=21, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:52,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_0_-false in 789 msec 2024-11-09T02:18:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-09T02:18:52,297 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_0_-false, procId: 21 completed 2024-11-09T02:18:52,297 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:52,297 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:52,302 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:52,302 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:52,303 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_0_-false assigned. 2024-11-09T02:18:52,305 DEBUG [hconnection-0x62c00555-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:52,308 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60932, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:52,319 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:52,322 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:18:52,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:18:52,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T02:18:52,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-09T02:18:52,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:52,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-09T02:18:52,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:18:53,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T02:18:54,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:18:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:60932 deadline: 1731118735325, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:18:54,347 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:18:54,348 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269736 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:18:54,348 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2025 2024-11-09T02:18:54,351 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d52f25d to 127.0.0.1:57725 2024-11-09T02:18:54,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:54,352 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:54,352 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:54,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:54,352 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a9c4477 to 127.0.0.1:57725 2024-11-09T02:18:54,371 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[0] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=456 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 384), ProcessCount=11 (was 11), AvailableMemoryMB=10025 (was 10068) 2024-11-09T02:18:54,387 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=227, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=10023 2024-11-09T02:18:54,390 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41a77621 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69ad79a5 2024-11-09T02:18:54,395 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x41a77621 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c52110e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:54,397 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69e88404 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3efb3040 2024-11-09T02:18:54,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1012c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:54,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:18:54,410 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:54,411 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:54,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_0_-false" procId is: 24 2024-11-09T02:18:54,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T02:18:54,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741847_1023 (size=427) 2024-11-09T02:18:54,435 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ce03aa7d3b74ad9aaf2d966daa46e587, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:54,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741848_1024 (size=78) 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ce03aa7d3b74ad9aaf2d966daa46e587, disabling compactions & flushes 2024-11-09T02:18:54,463 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. after waiting 0 ms 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,463 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,463 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ce03aa7d3b74ad9aaf2d966daa46e587: 2024-11-09T02:18:54,465 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:54,466 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731118734465"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118734465"}]},"ts":"1731118734465"} 2024-11-09T02:18:54,469 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:54,471 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:54,471 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118734471"}]},"ts":"1731118734471"} 2024-11-09T02:18:54,476 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:54,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce03aa7d3b74ad9aaf2d966daa46e587, ASSIGN}] 2024-11-09T02:18:54,488 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce03aa7d3b74ad9aaf2d966daa46e587, ASSIGN 2024-11-09T02:18:54,490 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce03aa7d3b74ad9aaf2d966daa46e587, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:54,641 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=ce03aa7d3b74ad9aaf2d966daa46e587, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:54,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE; OpenRegionProcedure ce03aa7d3b74ad9aaf2d966daa46e587, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:54,768 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:18:54,797 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:54,804 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,805 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7285): Opening region: {ENCODED => ce03aa7d3b74ad9aaf2d966daa46e587, NAME => 'testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:54,805 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_0_-false ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,805 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:54,805 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7327): checking encryption for ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,806 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7330): checking classloading for ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,809 INFO [StoreOpener-ce03aa7d3b74ad9aaf2d966daa46e587-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,814 INFO [StoreOpener-ce03aa7d3b74ad9aaf2d966daa46e587-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce03aa7d3b74ad9aaf2d966daa46e587 columnFamilyName testFamily 2024-11-09T02:18:54,814 DEBUG [StoreOpener-ce03aa7d3b74ad9aaf2d966daa46e587-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:54,815 INFO [StoreOpener-ce03aa7d3b74ad9aaf2d966daa46e587-1 {}] regionserver.HStore(327): Store=ce03aa7d3b74ad9aaf2d966daa46e587/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:54,819 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,821 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,828 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1085): writing seq id for ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:18:54,842 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_0_-false' 2024-11-09T02:18:54,843 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_0_-false' 2024-11-09T02:18:54,849 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:54,852 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1102): Opened ce03aa7d3b74ad9aaf2d966daa46e587; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70784008, jitterRate=0.05476391315460205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:54,853 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1001): Region open journal for ce03aa7d3b74ad9aaf2d966daa46e587: 2024-11-09T02:18:54,855 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587., pid=26, masterSystemTime=1731118734797 2024-11-09T02:18:54,858 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,858 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:18:54,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=25 updating hbase:meta row=ce03aa7d3b74ad9aaf2d966daa46e587, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:54,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T02:18:54,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, ppid=25, state=SUCCESS; OpenRegionProcedure ce03aa7d3b74ad9aaf2d966daa46e587, server=54f3333e9615,45997,1731118672827 in 218 msec 2024-11-09T02:18:54,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-09T02:18:54,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false, region=ce03aa7d3b74ad9aaf2d966daa46e587, ASSIGN in 385 msec 2024-11-09T02:18:54,872 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:54,872 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118734872"}]},"ts":"1731118734872"} 2024-11-09T02:18:54,874 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:54,881 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:54,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_0_-false in 475 msec 2024-11-09T02:18:55,388 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:18:55,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T02:18:55,417 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_0_-false, procId: 24 completed 2024-11-09T02:18:55,417 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:55,417 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:55,438 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:55,439 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:55,439 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_0_-false assigned. 2024-11-09T02:18:55,448 DEBUG [hconnection-0x74d9be8b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:55,451 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60934, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:55,470 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:55,475 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:18:55,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:18:56,539 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:18:56,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T02:18:57,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:18:58,485 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5025: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5025: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:18:58,486 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T02:18:58.486Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5025: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5025: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms row '' on table 'testNormalScanTimeoutOnOpenScanner_0_-false' at region=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=994ms, rpcTimeout=984ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:18:58,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3011 2024-11-09T02:18:58,487 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69e88404 to 127.0.0.1:57725 2024-11-09T02:18:58,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:58,487 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:18:58,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:18:58,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:18:58,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x41a77621 to 127.0.0.1:57725 2024-11-09T02:18:58,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[0] Thread=226 (was 227), OpenFileDescriptor=456 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 361), ProcessCount=11 (was 11), AvailableMemoryMB=10006 (was 10023) 2024-11-09T02:18:58,515 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=226, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=9997 2024-11-09T02:18:58,518 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d532004 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d0541ea 2024-11-09T02:18:58,523 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x7d532004 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f19dbbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:58,525 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50ca2cd2 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29e3c0fe 2024-11-09T02:18:58,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd486c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:18:58,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:18:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T02:18:58,537 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:18:58,537 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:58,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_0_-false" procId is: 27 2024-11-09T02:18:58,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:18:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-09T02:18:58,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741849_1025 (size=425) 2024-11-09T02:18:58,577 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 1e9b57e787ebaa31f7960f0e0be0fb6f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:18:58,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741850_1026 (size=76) 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 1e9b57e787ebaa31f7960f0e0be0fb6f, disabling compactions & flushes 2024-11-09T02:18:58,616 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. after waiting 0 ms 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,616 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 1e9b57e787ebaa31f7960f0e0be0fb6f: 2024-11-09T02:18:58,619 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:18:58,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731118738619"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118738619"}]},"ts":"1731118738619"} 2024-11-09T02:18:58,622 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:18:58,627 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:18:58,627 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118738627"}]},"ts":"1731118738627"} 2024-11-09T02:18:58,631 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:18:58,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=1e9b57e787ebaa31f7960f0e0be0fb6f, ASSIGN}] 2024-11-09T02:18:58,639 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=1e9b57e787ebaa31f7960f0e0be0fb6f, ASSIGN 2024-11-09T02:18:58,640 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=28, ppid=27, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=1e9b57e787ebaa31f7960f0e0be0fb6f, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:18:58,790 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=1e9b57e787ebaa31f7960f0e0be0fb6f, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:58,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; OpenRegionProcedure 1e9b57e787ebaa31f7960f0e0be0fb6f, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:18:58,948 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:18:58,954 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,954 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7285): Opening region: {ENCODED => 1e9b57e787ebaa31f7960f0e0be0fb6f, NAME => 'testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:18:58,955 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_0_-false 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,955 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:18:58,955 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7327): checking encryption for 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,955 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7330): checking classloading for 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,957 INFO [StoreOpener-1e9b57e787ebaa31f7960f0e0be0fb6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,959 INFO [StoreOpener-1e9b57e787ebaa31f7960f0e0be0fb6f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e9b57e787ebaa31f7960f0e0be0fb6f columnFamilyName testFamily 2024-11-09T02:18:58,959 DEBUG [StoreOpener-1e9b57e787ebaa31f7960f0e0be0fb6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:18:58,961 INFO [StoreOpener-1e9b57e787ebaa31f7960f0e0be0fb6f-1 {}] regionserver.HStore(327): Store=1e9b57e787ebaa31f7960f0e0be0fb6f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:18:58,962 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,963 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,967 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1085): writing seq id for 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:18:58,972 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:18:58,973 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1102): Opened 1e9b57e787ebaa31f7960f0e0be0fb6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59177613, jitterRate=-0.11818484961986542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:18:58,975 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1001): Region open journal for 1e9b57e787ebaa31f7960f0e0be0fb6f: 2024-11-09T02:18:58,977 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f., pid=29, masterSystemTime=1731118738947 2024-11-09T02:18:58,979 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,979 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:18:58,980 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=28 updating hbase:meta row=1e9b57e787ebaa31f7960f0e0be0fb6f, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:18:58,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T02:18:58,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; OpenRegionProcedure 1e9b57e787ebaa31f7960f0e0be0fb6f, server=54f3333e9615,45997,1731118672827 in 189 msec 2024-11-09T02:18:58,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T02:18:58,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_0_-false, region=1e9b57e787ebaa31f7960f0e0be0fb6f, ASSIGN in 349 msec 2024-11-09T02:18:58,990 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:18:58,990 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118738990"}]},"ts":"1731118738990"} 2024-11-09T02:18:58,993 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:18:58,993 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:18:58,997 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=27, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:18:59,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_0_-false in 464 msec 2024-11-09T02:18:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-09T02:18:59,543 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_0_-false, procId: 27 completed 2024-11-09T02:18:59,543 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:18:59,543 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:59,548 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T02:18:59,548 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:18:59,548 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_0_-false assigned. 2024-11-09T02:18:59,551 DEBUG [hconnection-0x259c0df0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:59,554 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60940, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:59,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:18:59,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:18:59,573 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:18:59,581 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x7d532004 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:18:59,581 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x7d532004 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:18:59,590 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60948, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:18:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:18:59,595 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:18:59,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269748 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:59,596 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:18:59,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269748 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:18:59,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T02:19:00,845 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_0_-false' 2024-11-09T02:19:00,845 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_0_-false' 2024-11-09T02:19:02,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T02:19:02,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:02,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:19:02,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:02,605 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3009 2024-11-09T02:19:02,605 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50ca2cd2 to 127.0.0.1:57725 2024-11-09T02:19:02,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:02,606 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:02,606 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:02,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:02,606 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269748 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:02,606 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d532004 to 127.0.0.1:57725 2024-11-09T02:19:02,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281868' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T02:19:02,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:60948 deadline: 1731118743605, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281868' does not exist 2024-11-09T02:19:02,606 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 54f3333e9615,45997,1731118672827 for closing scanner id = -1481684882995281868 for 1e9b57e787ebaa31f7960f0e0be0fb6f of testNormalScanTimeoutOnNextAsync_0_-false failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T02:19:02,610 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:02,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[0] Thread=227 (was 226) - Thread LEAK? -, OpenFileDescriptor=454 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 361) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9989 (was 9997) 2024-11-09T02:19:02,639 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=227, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=9989 2024-11-09T02:19:02,642 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6676292d to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@547adcc4 2024-11-09T02:19:02,671 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x6676292d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f62eaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:02,673 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x36cd47d5 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a1dbdc4 2024-11-09T02:19:02,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d75d0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:02,682 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:02,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false 2024-11-09T02:19:02,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:02,687 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:02,689 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:02,700 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_0_-false" procId is: 30 2024-11-09T02:19:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T02:19:02,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741851_1027 (size=417) 2024-11-09T02:19:03,098 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:03,130 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8efa319d73410a0150970c2107175f31, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:03,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741852_1028 (size=68) 2024-11-09T02:19:03,147 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:03,148 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 8efa319d73410a0150970c2107175f31, disabling compactions & flushes 2024-11-09T02:19:03,148 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,148 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,148 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. after waiting 0 ms 2024-11-09T02:19:03,148 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,148 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,148 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8efa319d73410a0150970c2107175f31: 2024-11-09T02:19:03,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:03,150 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31.","families":{"info":[{"qualifier":"regioninfo","vlen":67,"tag":[],"timestamp":"1731118743150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118743150"}]},"ts":"1731118743150"} 2024-11-09T02:19:03,154 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:03,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:03,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118743155"}]},"ts":"1731118743155"} 2024-11-09T02:19:03,158 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:03,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=8efa319d73410a0150970c2107175f31, ASSIGN}] 2024-11-09T02:19:03,166 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=8efa319d73410a0150970c2107175f31, ASSIGN 2024-11-09T02:19:03,167 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=8efa319d73410a0150970c2107175f31, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:03,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=8efa319d73410a0150970c2107175f31, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:03,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; OpenRegionProcedure 8efa319d73410a0150970c2107175f31, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:03,476 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:03,483 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7285): Opening region: {ENCODED => 8efa319d73410a0150970c2107175f31, NAME => 'hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:03,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_0_-false 8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:03,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7327): checking encryption for 8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(7330): checking classloading for 8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,487 INFO [StoreOpener-8efa319d73410a0150970c2107175f31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,490 INFO [StoreOpener-8efa319d73410a0150970c2107175f31-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8efa319d73410a0150970c2107175f31 columnFamilyName testFamily 2024-11-09T02:19:03,490 DEBUG [StoreOpener-8efa319d73410a0150970c2107175f31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:03,491 INFO [StoreOpener-8efa319d73410a0150970c2107175f31-1 {}] regionserver.HStore(327): Store=8efa319d73410a0150970c2107175f31/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:03,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,497 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1085): writing seq id for 8efa319d73410a0150970c2107175f31 2024-11-09T02:19:03,501 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:03,502 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1102): Opened 8efa319d73410a0150970c2107175f31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70810449, jitterRate=0.05515791475772858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:03,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegion(1001): Region open journal for 8efa319d73410a0150970c2107175f31: 2024-11-09T02:19:03,505 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31., pid=32, masterSystemTime=1731118743476 2024-11-09T02:19:03,508 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,508 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=32}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:19:03,509 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=8efa319d73410a0150970c2107175f31, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:03,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T02:19:03,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; OpenRegionProcedure 8efa319d73410a0150970c2107175f31, server=54f3333e9615,45997,1731118672827 in 193 msec 2024-11-09T02:19:03,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T02:19:03,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false, region=8efa319d73410a0150970c2107175f31, ASSIGN in 355 msec 2024-11-09T02:19:03,524 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:03,524 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118743524"}]},"ts":"1731118743524"} 2024-11-09T02:19:03,529 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:03,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:03,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_0_-false in 850 msec 2024-11-09T02:19:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T02:19:03,706 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_0_-false, procId: 30 completed 2024-11-09T02:19:03,706 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:03,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:03,721 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:03,721 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:03,721 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_0_-false assigned. 2024-11-09T02:19:03,727 DEBUG [hconnection-0x751b670-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:03,744 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47688, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:03,765 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:03,772 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:19:03,777 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269752 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:03,778 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-09T02:19:04,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:06,567 INFO [master/54f3333e9615:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T02:19:06,567 INFO [master/54f3333e9615:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T02:19:06,847 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_0_-false' 2024-11-09T02:19:10,278 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6506 2024-11-09T02:19:10,279 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x36cd47d5 to 127.0.0.1:57725 2024-11-09T02:19:10,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:10,279 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:10,279 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:10,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:10,279 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6676292d to 127.0.0.1:57725 2024-11-09T02:19:10,292 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[0] Thread=228 (was 227) - Thread LEAK? -, OpenFileDescriptor=452 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 364) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9960 (was 9989) 2024-11-09T02:19:10,304 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=228, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=9959 2024-11-09T02:19:10,307 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6587f449 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32bd74c9 2024-11-09T02:19:10,316 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x6587f449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578b5754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:10,320 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32cfbf1f to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@299eb4a0 2024-11-09T02:19:10,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e13866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:10,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false 2024-11-09T02:19:10,337 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:10,337 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:10,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_0_-false" procId is: 33 2024-11-09T02:19:10,338 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-09T02:19:10,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741853_1029 (size=421) 2024-11-09T02:19:10,761 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => fd148266e0f7adabaf27b8fe8e091c94, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:10,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741854_1030 (size=72) 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing fd148266e0f7adabaf27b8fe8e091c94, disabling compactions & flushes 2024-11-09T02:19:10,784 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. after waiting 0 ms 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:10,784 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:10,784 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for fd148266e0f7adabaf27b8fe8e091c94: 2024-11-09T02:19:10,787 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:10,787 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731118750787"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118750787"}]},"ts":"1731118750787"} 2024-11-09T02:19:10,790 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:10,793 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:10,793 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118750793"}]},"ts":"1731118750793"} 2024-11-09T02:19:10,796 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:10,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=fd148266e0f7adabaf27b8fe8e091c94, ASSIGN}] 2024-11-09T02:19:10,803 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=fd148266e0f7adabaf27b8fe8e091c94, ASSIGN 2024-11-09T02:19:10,804 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=fd148266e0f7adabaf27b8fe8e091c94, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:10,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=fd148266e0f7adabaf27b8fe8e091c94, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:10,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; OpenRegionProcedure fd148266e0f7adabaf27b8fe8e091c94, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:11,111 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:11,117 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:11,118 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7285): Opening region: {ENCODED => fd148266e0f7adabaf27b8fe8e091c94, NAME => 'testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:11,118 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_0_-false fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,118 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:11,118 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7327): checking encryption for fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,119 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7330): checking classloading for fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,121 INFO [StoreOpener-fd148266e0f7adabaf27b8fe8e091c94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,123 INFO [StoreOpener-fd148266e0f7adabaf27b8fe8e091c94-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd148266e0f7adabaf27b8fe8e091c94 columnFamilyName testFamily 2024-11-09T02:19:11,123 DEBUG [StoreOpener-fd148266e0f7adabaf27b8fe8e091c94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:11,123 INFO [StoreOpener-fd148266e0f7adabaf27b8fe8e091c94-1 {}] regionserver.HStore(327): Store=fd148266e0f7adabaf27b8fe8e091c94/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:11,125 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,125 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,128 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1085): writing seq id for fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:19:11,131 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:11,133 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1102): Opened fd148266e0f7adabaf27b8fe8e091c94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67777691, jitterRate=0.009966298937797546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:11,134 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1001): Region open journal for fd148266e0f7adabaf27b8fe8e091c94: 2024-11-09T02:19:11,135 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94., pid=35, masterSystemTime=1731118751111 2024-11-09T02:19:11,138 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:11,138 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:19:11,139 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=fd148266e0f7adabaf27b8fe8e091c94, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:11,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T02:19:11,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; OpenRegionProcedure fd148266e0f7adabaf27b8fe8e091c94, server=54f3333e9615,45997,1731118672827 in 183 msec 2024-11-09T02:19:11,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T02:19:11,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_0_-false, region=fd148266e0f7adabaf27b8fe8e091c94, ASSIGN in 344 msec 2024-11-09T02:19:11,147 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:11,148 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118751147"}]},"ts":"1731118751147"} 2024-11-09T02:19:11,150 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:11,154 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=33, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:11,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_0_-false in 821 msec 2024-11-09T02:19:11,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-09T02:19:11,345 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_0_-false, procId: 33 completed 2024-11-09T02:19:11,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:11,345 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:11,353 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:11,354 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:11,354 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_0_-false assigned. 2024-11-09T02:19:11,356 DEBUG [hconnection-0x43e4da05-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:11,359 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42612, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:11,373 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:11,377 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:19:11,380 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269756 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:11,381 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-09T02:19:12,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-09T02:19:12,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:12,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-09T02:19:12,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:12,848 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_0_-false' 2024-11-09T02:19:12,882 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1504 2024-11-09T02:19:12,882 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32cfbf1f to 127.0.0.1:57725 2024-11-09T02:19:12,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:12,882 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:12,883 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:12,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:12,883 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6587f449 to 127.0.0.1:57725 2024-11-09T02:19:12,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[0] Thread=231 (was 228) - Thread LEAK? -, OpenFileDescriptor=454 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 369), ProcessCount=11 (was 11), AvailableMemoryMB=9953 (was 9959) 2024-11-09T02:19:12,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=231, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=9953 2024-11-09T02:19:12,908 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40d2969b to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e0695f3 2024-11-09T02:19:12,911 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x40d2969b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24181547, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:12,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56a767c0 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c771504 2024-11-09T02:19:12,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c52a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:12,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false 2024-11-09T02:19:12,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:12,924 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:12,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_0_-false" procId is: 36 2024-11-09T02:19:12,925 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T02:19:12,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741855_1031 (size=420) 2024-11-09T02:19:12,944 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 92e0494411be1d7676e60e07472411c6, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741856_1032 (size=71) 2024-11-09T02:19:12,959 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:12,959 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 92e0494411be1d7676e60e07472411c6, disabling compactions & flushes 2024-11-09T02:19:12,959 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:12,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:12,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. after waiting 0 ms 2024-11-09T02:19:12,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:12,960 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:12,960 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 92e0494411be1d7676e60e07472411c6: 2024-11-09T02:19:12,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:12,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731118752961"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118752961"}]},"ts":"1731118752961"} 2024-11-09T02:19:12,964 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:12,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:12,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118752966"}]},"ts":"1731118752966"} 2024-11-09T02:19:12,969 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:12,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=92e0494411be1d7676e60e07472411c6, ASSIGN}] 2024-11-09T02:19:12,982 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=92e0494411be1d7676e60e07472411c6, ASSIGN 2024-11-09T02:19:12,983 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=92e0494411be1d7676e60e07472411c6, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:13,134 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=92e0494411be1d7676e60e07472411c6, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:13,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; OpenRegionProcedure 92e0494411be1d7676e60e07472411c6, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:13,289 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:13,295 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:13,295 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7285): Opening region: {ENCODED => 92e0494411be1d7676e60e07472411c6, NAME => 'testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:13,296 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_0_-false 92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,296 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:13,296 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7327): checking encryption for 92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,296 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7330): checking classloading for 92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,305 INFO [StoreOpener-92e0494411be1d7676e60e07472411c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,307 INFO [StoreOpener-92e0494411be1d7676e60e07472411c6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92e0494411be1d7676e60e07472411c6 columnFamilyName testFamily 2024-11-09T02:19:13,307 DEBUG [StoreOpener-92e0494411be1d7676e60e07472411c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:13,309 INFO [StoreOpener-92e0494411be1d7676e60e07472411c6-1 {}] regionserver.HStore(327): Store=92e0494411be1d7676e60e07472411c6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:13,310 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,311 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,318 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1085): writing seq id for 92e0494411be1d7676e60e07472411c6 2024-11-09T02:19:13,325 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:13,325 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1102): Opened 92e0494411be1d7676e60e07472411c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71301514, jitterRate=0.062475353479385376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:13,326 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1001): Region open journal for 92e0494411be1d7676e60e07472411c6: 2024-11-09T02:19:13,328 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6., pid=38, masterSystemTime=1731118753289 2024-11-09T02:19:13,330 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:13,330 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:19:13,331 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=37 updating hbase:meta row=92e0494411be1d7676e60e07472411c6, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:13,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-09T02:19:13,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; OpenRegionProcedure 92e0494411be1d7676e60e07472411c6, server=54f3333e9615,45997,1731118672827 in 197 msec 2024-11-09T02:19:13,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-09T02:19:13,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_0_-false, region=92e0494411be1d7676e60e07472411c6, ASSIGN in 357 msec 2024-11-09T02:19:13,341 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:13,341 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118753341"}]},"ts":"1731118753341"} 2024-11-09T02:19:13,343 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:13,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:13,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_0_-false in 425 msec 2024-11-09T02:19:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T02:19:13,929 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_0_-false, procId: 36 completed 2024-11-09T02:19:13,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:13,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:13,935 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:13,935 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:13,935 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_0_-false assigned. 2024-11-09T02:19:13,938 DEBUG [hconnection-0x5a8fcc67-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:13,940 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:13,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:13,956 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x485e828d to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f06d652 2024-11-09T02:19:13,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf1ade8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:13,966 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:19:13,969 DEBUG [hconnection-0x4649b214-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:13,973 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42630, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:13,979 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:19:13,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269761 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:13,981 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:19:13,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269761 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:13,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T02:19:14,985 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T02:19:14.985Z, RpcRetryingCaller{globalStartTime=2024-11-09T02:19:13.981Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T02:19:14.985Z, RpcRetryingCaller{globalStartTime=2024-11-09T02:19:13.981Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:19:14,985 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1004 2024-11-09T02:19:14,985 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x485e828d to 127.0.0.1:57725 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x56a767c0 to 127.0.0.1:57725 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:14,986 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:14,986 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x40d2969b to 127.0.0.1:57725 2024-11-09T02:19:14,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[0] Thread=236 (was 231) - Thread LEAK? -, OpenFileDescriptor=459 (was 454) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 369) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9950 (was 9953) 2024-11-09T02:19:15,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=236, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=9949 2024-11-09T02:19:15,005 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x229c45cb to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@128c8246 2024-11-09T02:19:15,009 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x229c45cb {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6141fff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:15,010 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x160d2d2e to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f61853b 2024-11-09T02:19:15,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f933301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:15,015 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:15,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T02:19:15,018 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:15,018 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:15,018 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_0_-false" procId is: 39 2024-11-09T02:19:15,019 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T02:19:15,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741857_1033 (size=422) 2024-11-09T02:19:15,429 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2ac227a2fa5b5a4d941fcbac940cb5fb, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:15,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741858_1034 (size=73) 2024-11-09T02:19:15,483 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:15,837 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:15,838 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 2ac227a2fa5b5a4d941fcbac940cb5fb, disabling compactions & flushes 2024-11-09T02:19:15,838 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:15,838 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:15,838 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. after waiting 0 ms 2024-11-09T02:19:15,838 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:15,838 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:15,838 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2ac227a2fa5b5a4d941fcbac940cb5fb: 2024-11-09T02:19:15,839 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:15,840 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731118755839"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118755839"}]},"ts":"1731118755839"} 2024-11-09T02:19:15,842 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:15,843 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:15,843 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118755843"}]},"ts":"1731118755843"} 2024-11-09T02:19:15,845 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:15,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2ac227a2fa5b5a4d941fcbac940cb5fb, ASSIGN}] 2024-11-09T02:19:15,850 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2ac227a2fa5b5a4d941fcbac940cb5fb, ASSIGN 2024-11-09T02:19:15,850 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2ac227a2fa5b5a4d941fcbac940cb5fb, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:16,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=2ac227a2fa5b5a4d941fcbac940cb5fb, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:16,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 2ac227a2fa5b5a4d941fcbac940cb5fb, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T02:19:16,155 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:16,160 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:16,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7285): Opening region: {ENCODED => 2ac227a2fa5b5a4d941fcbac940cb5fb, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:16,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_0_-false 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:16,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7327): checking encryption for 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,162 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(7330): checking classloading for 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,164 INFO [StoreOpener-2ac227a2fa5b5a4d941fcbac940cb5fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,165 INFO [StoreOpener-2ac227a2fa5b5a4d941fcbac940cb5fb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ac227a2fa5b5a4d941fcbac940cb5fb columnFamilyName testFamily 2024-11-09T02:19:16,165 DEBUG [StoreOpener-2ac227a2fa5b5a4d941fcbac940cb5fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:16,166 INFO [StoreOpener-2ac227a2fa5b5a4d941fcbac940cb5fb-1 {}] regionserver.HStore(327): Store=2ac227a2fa5b5a4d941fcbac940cb5fb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:16,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,170 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1085): writing seq id for 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:19:16,172 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:16,173 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1102): Opened 2ac227a2fa5b5a4d941fcbac940cb5fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65180385, jitterRate=-0.028736576437950134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:16,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegion(1001): Region open journal for 2ac227a2fa5b5a4d941fcbac940cb5fb: 2024-11-09T02:19:16,175 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb., pid=41, masterSystemTime=1731118756155 2024-11-09T02:19:16,177 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:16,177 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=41}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:19:16,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=2ac227a2fa5b5a4d941fcbac940cb5fb, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:16,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T02:19:16,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 2ac227a2fa5b5a4d941fcbac940cb5fb, server=54f3333e9615,45997,1731118672827 in 176 msec 2024-11-09T02:19:16,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-09T02:19:16,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false, region=2ac227a2fa5b5a4d941fcbac940cb5fb, ASSIGN in 332 msec 2024-11-09T02:19:16,183 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:16,184 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118756183"}]},"ts":"1731118756183"} 2024-11-09T02:19:16,185 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:16,188 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=39, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:16,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_0_-false in 1.1730 sec 2024-11-09T02:19:16,645 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:18,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-09T02:19:18,023 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_0_-false, procId: 39 completed 2024-11-09T02:19:18,023 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:18,023 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:18,027 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:18,028 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:18,028 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_0_-false assigned. 2024-11-09T02:19:18,030 DEBUG [hconnection-0x53f60566-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:18,031 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42644, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:18,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:18,042 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:19:18,044 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:19:18,044 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T02:19:18,849 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_0_-false' 2024-11-09T02:19:18,850 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_0_-false' 2024-11-09T02:19:21,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:19:21,663 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:22,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-09T02:19:22,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:22,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T02:19:22,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T02:19:24,853 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:19:25,058 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:19:25,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:42644 deadline: 1731118767044, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:19:25,059 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:19:25,059 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269765 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:19:25,060 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7018 2024-11-09T02:19:25,060 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x160d2d2e to 127.0.0.1:57725 2024-11-09T02:19:25,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:25,060 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:25,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:25,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:25,061 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x229c45cb to 127.0.0.1:57725 2024-11-09T02:19:25,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[0] Thread=230 (was 236), OpenFileDescriptor=454 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=9938 (was 9949) 2024-11-09T02:19:25,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=230, OpenFileDescriptor=454, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=9937 2024-11-09T02:19:25,088 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x523c9018 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6cba182f 2024-11-09T02:19:25,099 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x523c9018 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fef0af1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:25,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d0e27e7 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4af42166 2024-11-09T02:19:25,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7ec470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:25,109 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:25,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:19:25,112 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:25,112 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:25,112 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_0_-false" procId is: 42 2024-11-09T02:19:25,113 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-09T02:19:25,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741859_1035 (size=428) 2024-11-09T02:19:25,130 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3152c4d9a734cd45a64adc4eaba61986, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741860_1036 (size=79) 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 3152c4d9a734cd45a64adc4eaba61986, disabling compactions & flushes 2024-11-09T02:19:25,146 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. after waiting 0 ms 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,146 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,146 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3152c4d9a734cd45a64adc4eaba61986: 2024-11-09T02:19:25,148 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:25,148 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731118765148"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118765148"}]},"ts":"1731118765148"} 2024-11-09T02:19:25,151 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:25,152 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:25,152 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118765152"}]},"ts":"1731118765152"} 2024-11-09T02:19:25,154 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:25,158 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=3152c4d9a734cd45a64adc4eaba61986, ASSIGN}] 2024-11-09T02:19:25,159 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=3152c4d9a734cd45a64adc4eaba61986, ASSIGN 2024-11-09T02:19:25,160 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=3152c4d9a734cd45a64adc4eaba61986, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:25,310 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=3152c4d9a734cd45a64adc4eaba61986, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:25,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; OpenRegionProcedure 3152c4d9a734cd45a64adc4eaba61986, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:25,465 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:25,471 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 3152c4d9a734cd45a64adc4eaba61986, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:25,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_0_-false 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:25,472 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,473 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,474 INFO [StoreOpener-3152c4d9a734cd45a64adc4eaba61986-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,477 INFO [StoreOpener-3152c4d9a734cd45a64adc4eaba61986-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3152c4d9a734cd45a64adc4eaba61986 columnFamilyName testFamily 2024-11-09T02:19:25,477 DEBUG [StoreOpener-3152c4d9a734cd45a64adc4eaba61986-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:25,477 INFO [StoreOpener-3152c4d9a734cd45a64adc4eaba61986-1 {}] regionserver.HStore(327): Store=3152c4d9a734cd45a64adc4eaba61986/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:25,479 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,479 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:19:25,488 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:25,489 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1102): Opened 3152c4d9a734cd45a64adc4eaba61986; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65200134, jitterRate=-0.028442293405532837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:25,490 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 3152c4d9a734cd45a64adc4eaba61986: 2024-11-09T02:19:25,491 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., pid=44, masterSystemTime=1731118765465 2024-11-09T02:19:25,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,493 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=44}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:19:25,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=43 updating hbase:meta row=3152c4d9a734cd45a64adc4eaba61986, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:25,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-09T02:19:25,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; OpenRegionProcedure 3152c4d9a734cd45a64adc4eaba61986, server=54f3333e9615,45997,1731118672827 in 183 msec 2024-11-09T02:19:25,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T02:19:25,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, region=3152c4d9a734cd45a64adc4eaba61986, ASSIGN in 341 msec 2024-11-09T02:19:25,503 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:25,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118765503"}]},"ts":"1731118765503"} 2024-11-09T02:19:25,505 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:25,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=42, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:25,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false in 398 msec 2024-11-09T02:19:26,046 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:19:26,051 DEBUG [master/54f3333e9615:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 6235bfc49272e5114ec5c10328305fa2 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:19:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=42 2024-11-09T02:19:26,122 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false, procId: 42 completed 2024-11-09T02:19:26,122 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:26,122 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:26,126 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:26,127 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:26,127 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-09T02:19:26,129 DEBUG [hconnection-0x48186989-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:26,131 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51306, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:26,141 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:26,145 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:19:26,145 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:19:26,146 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x523c9018 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:19:26,147 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x523c9018 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:26,148 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51314, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:19:26,150 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:19:28,551 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:19:30,974 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:31,012 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-09T02:19:32,156 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-09T02:19:32,156 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-09T02:19:32,156 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:19:32,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:19:32,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T02:19:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:19:33,167 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:19:35,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms 2024-11-09T02:19:35,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms 2024-11-09T02:19:35,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:19:35,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T02:19:32.156Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6005ms, rpcTimeout=6000ms 2024-11-09T02:19:35.156Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1989ms, rpcTimeout=1979ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T02:19:35,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9012 2024-11-09T02:19:35,157 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d0e27e7 to 127.0.0.1:57725 2024-11-09T02:19:35,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:35,158 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:35,158 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:35,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:35,158 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x523c9018 to 127.0.0.1:57725 2024-11-09T02:19:35,168 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[0] Thread=230 (was 230), OpenFileDescriptor=448 (was 454), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 345) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9920 (was 9937) 2024-11-09T02:19:35,178 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=9919 2024-11-09T02:19:35,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e65352b to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79458f02 2024-11-09T02:19:35,184 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x6e65352b {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6c9c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:35,186 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x656c684a to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@146becc0 2024-11-09T02:19:35,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53604f0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:35,192 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:35,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false 2024-11-09T02:19:35,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:35,194 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:35,195 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_0_-false" procId is: 45 2024-11-09T02:19:35,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T02:19:35,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741861_1037 (size=416) 2024-11-09T02:19:35,587 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281900 expired clientIPAndPort=172.17.0.2:35874, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:19:35,617 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 755bcccae28504c223e008c40b24c6c4, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:35,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741862_1038 (size=67) 2024-11-09T02:19:36,026 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:36,027 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing 755bcccae28504c223e008c40b24c6c4, disabling compactions & flushes 2024-11-09T02:19:36,027 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,027 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,027 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. after waiting 0 ms 2024-11-09T02:19:36,027 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,027 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,027 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for 755bcccae28504c223e008c40b24c6c4: 2024-11-09T02:19:36,028 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:36,029 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731118776028"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118776028"}]},"ts":"1731118776028"} 2024-11-09T02:19:36,031 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:36,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:36,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118776032"}]},"ts":"1731118776032"} 2024-11-09T02:19:36,034 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:36,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=755bcccae28504c223e008c40b24c6c4, ASSIGN}] 2024-11-09T02:19:36,040 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=755bcccae28504c223e008c40b24c6c4, ASSIGN 2024-11-09T02:19:36,040 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=755bcccae28504c223e008c40b24c6c4, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:36,191 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=755bcccae28504c223e008c40b24c6c4, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:36,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; OpenRegionProcedure 755bcccae28504c223e008c40b24c6c4, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T02:19:36,346 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:36,358 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,358 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7285): Opening region: {ENCODED => 755bcccae28504c223e008c40b24c6c4, NAME => 'hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:36,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_0_-false 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:36,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7327): checking encryption for 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,359 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(7330): checking classloading for 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,364 INFO [StoreOpener-755bcccae28504c223e008c40b24c6c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,366 INFO [StoreOpener-755bcccae28504c223e008c40b24c6c4-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 755bcccae28504c223e008c40b24c6c4 columnFamilyName testFamily 2024-11-09T02:19:36,367 DEBUG [StoreOpener-755bcccae28504c223e008c40b24c6c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:36,367 INFO [StoreOpener-755bcccae28504c223e008c40b24c6c4-1 {}] regionserver.HStore(327): Store=755bcccae28504c223e008c40b24c6c4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:36,368 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,369 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,374 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1085): writing seq id for 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:19:36,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:36,381 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1102): Opened 755bcccae28504c223e008c40b24c6c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64802968, jitterRate=-0.034360527992248535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:36,383 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegion(1001): Region open journal for 755bcccae28504c223e008c40b24c6c4: 2024-11-09T02:19:36,384 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4., pid=47, masterSystemTime=1731118776346 2024-11-09T02:19:36,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,386 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=47}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:19:36,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=755bcccae28504c223e008c40b24c6c4, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:36,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T02:19:36,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, ppid=46, state=SUCCESS; OpenRegionProcedure 755bcccae28504c223e008c40b24c6c4, server=54f3333e9615,45997,1731118672827 in 195 msec 2024-11-09T02:19:36,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-09T02:19:36,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false, region=755bcccae28504c223e008c40b24c6c4, ASSIGN in 353 msec 2024-11-09T02:19:36,395 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:36,395 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118776395"}]},"ts":"1731118776395"} 2024-11-09T02:19:36,397 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:36,400 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=45, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:36,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_0_-false in 1.2090 sec 2024-11-09T02:19:37,015 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_0_-false' 2024-11-09T02:19:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T02:19:38,200 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_0_-false, procId: 45 completed 2024-11-09T02:19:38,201 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:38,201 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:38,206 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:38,206 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:38,206 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_0_-false assigned. 2024-11-09T02:19:38,212 DEBUG [hconnection-0x1a2f0601-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:38,214 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54772, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:38,225 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:38,229 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e9141a0 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a54b6c4 2024-11-09T02:19:38,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ab0a12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:38,234 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:19:38,236 DEBUG [hconnection-0x27367555-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:38,237 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54780, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:38,241 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:19:38,242 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269778 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:38,243 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:19:38,244 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269778 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:38,244 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T02:19:39,667 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:41,904 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-09T02:19:42,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:42,694 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281898 expired clientIPAndPort=172.17.0.2:35874, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:19:44,245 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T02:19:44.245Z, RpcRetryingCaller{globalStartTime=2024-11-09T02:19:38.244Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=1, exceptions: 2024-11-09T02:19:44.245Z, RpcRetryingCaller{globalStartTime=2024-11-09T02:19:38.244Z, pause=1000, maxAttempts=1}, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=6001ms, rpcTimeout=6000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:19:44,245 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6001 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e9141a0 to 127.0.0.1:57725 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x656c684a to 127.0.0.1:57725 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:44,246 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:44,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e65352b to 127.0.0.1:57725 2024-11-09T02:19:44,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[0] Thread=235 (was 230) - Thread LEAK? -, OpenFileDescriptor=451 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 354), ProcessCount=11 (was 11), AvailableMemoryMB=9906 (was 9919) 2024-11-09T02:19:44,264 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=235, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=9905 2024-11-09T02:19:44,266 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a6f8b19 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e0d59dd 2024-11-09T02:19:44,269 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2a6f8b19 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302aa3e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:44,270 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x039bfca8 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39634faf 2024-11-09T02:19:44,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f023cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:44,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:19:44,280 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:44,280 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:44,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_0_-false" procId is: 48 2024-11-09T02:19:44,281 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T02:19:44,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741863_1039 (size=432) 2024-11-09T02:19:44,692 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ff049946e3d00cdf907ccf7cd7ae4297, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_0_-false', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:44,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741864_1040 (size=83) 2024-11-09T02:19:44,745 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:45,100 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:45,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1681): Closing ff049946e3d00cdf907ccf7cd7ae4297, disabling compactions & flushes 2024-11-09T02:19:45,101 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. after waiting 0 ms 2024-11-09T02:19:45,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,101 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,101 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_0_-false-pool-0 {}] regionserver.HRegion(1635): Region close journal for ff049946e3d00cdf907ccf7cd7ae4297: 2024-11-09T02:19:45,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:45,102 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297.","families":{"info":[{"qualifier":"regioninfo","vlen":82,"tag":[],"timestamp":"1731118785102"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118785102"}]},"ts":"1731118785102"} 2024-11-09T02:19:45,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:45,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:45,105 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118785105"}]},"ts":"1731118785105"} 2024-11-09T02:19:45,107 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLING in hbase:meta 2024-11-09T02:19:45,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ff049946e3d00cdf907ccf7cd7ae4297, ASSIGN}] 2024-11-09T02:19:45,113 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ff049946e3d00cdf907ccf7cd7ae4297, ASSIGN 2024-11-09T02:19:45,113 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=49, ppid=48, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ff049946e3d00cdf907ccf7cd7ae4297, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:45,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=ff049946e3d00cdf907ccf7cd7ae4297, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:45,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; OpenRegionProcedure ff049946e3d00cdf907ccf7cd7ae4297, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T02:19:45,419 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:45,424 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,424 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7285): Opening region: {ENCODED => ff049946e3d00cdf907ccf7cd7ae4297, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:45,424 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_0_-false ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,424 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:45,424 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7327): checking encryption for ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,424 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7330): checking classloading for ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,426 INFO [StoreOpener-ff049946e3d00cdf907ccf7cd7ae4297-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,428 INFO [StoreOpener-ff049946e3d00cdf907ccf7cd7ae4297-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff049946e3d00cdf907ccf7cd7ae4297 columnFamilyName testFamily 2024-11-09T02:19:45,428 DEBUG [StoreOpener-ff049946e3d00cdf907ccf7cd7ae4297-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:45,428 INFO [StoreOpener-ff049946e3d00cdf907ccf7cd7ae4297-1 {}] regionserver.HStore(327): Store=ff049946e3d00cdf907ccf7cd7ae4297/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:45,429 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,430 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,432 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1085): writing seq id for ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:19:45,435 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:45,435 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1102): Opened ff049946e3d00cdf907ccf7cd7ae4297; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67152126, jitterRate=6.446540355682373E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:45,436 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1001): Region open journal for ff049946e3d00cdf907ccf7cd7ae4297: 2024-11-09T02:19:45,437 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., pid=50, masterSystemTime=1731118785419 2024-11-09T02:19:45,439 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,439 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:19:45,439 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=49 updating hbase:meta row=ff049946e3d00cdf907ccf7cd7ae4297, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:45,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-09T02:19:45,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; OpenRegionProcedure ff049946e3d00cdf907ccf7cd7ae4297, server=54f3333e9615,45997,1731118672827 in 175 msec 2024-11-09T02:19:45,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T02:19:45,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, ppid=48, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false, region=ff049946e3d00cdf907ccf7cd7ae4297, ASSIGN in 331 msec 2024-11-09T02:19:45,445 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:45,446 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_0_-false","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118785445"}]},"ts":"1731118785445"} 2024-11-09T02:19:45,447 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_0_-false, state=ENABLED in hbase:meta 2024-11-09T02:19:45,450 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=48, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:45,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_0_-false in 1.1720 sec 2024-11-09T02:19:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=48 2024-11-09T02:19:47,285 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_0_-false, procId: 48 completed 2024-11-09T02:19:47,285 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_0_-false get assigned. Timeout = 60000ms 2024-11-09T02:19:47,285 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:47,289 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned to meta. Checking AM states. 2024-11-09T02:19:47,290 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:47,290 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_0_-false assigned. 2024-11-09T02:19:47,292 DEBUG [hconnection-0xa20bde7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:47,293 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48408, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:47,301 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:47,303 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:19:47,304 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:19:47,305 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2a6f8b19 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:19:47,305 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x2a6f8b19 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:47,306 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48414, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:19:47,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:19:48,314 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T02:19:48,314 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T02:19:48,314 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:19:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T02:19:49,017 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_0_-false' 2024-11-09T02:19:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_0_-false', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:19:49,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:19:50,305 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=977ms 2024-11-09T02:19:50,305 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=977ms 2024-11-09T02:19:50,305 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:19:50,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T02:19:48.315Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T02:19:50.305Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=977ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=977ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=979ms, rpcTimeout=977ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T02:19:50,307 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3003 2024-11-09T02:19:50,307 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x039bfca8 to 127.0.0.1:57725 2024-11-09T02:19:50,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:50,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:50,307 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:50,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:50,308 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a6f8b19 to 127.0.0.1:57725 2024-11-09T02:19:50,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[0] Thread=231 (was 235), OpenFileDescriptor=448 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=9898 (was 9905) 2024-11-09T02:19:50,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=9897 2024-11-09T02:19:50,327 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32b3cd9e to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a47aa81 2024-11-09T02:19:50,330 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x32b3cd9e {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b609998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:50,331 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a4cc413 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2bcd6b30 2024-11-09T02:19:50,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be8cea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:50,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:50,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-09T02:19:50,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:50,339 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:50,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync_1_-true" procId is: 51 2024-11-09T02:19:50,340 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T02:19:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741865_1041 (size=436) 2024-11-09T02:19:50,701 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281893 expired clientIPAndPort=172.17.0.2:38284, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:19:50,749 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2452fe7575fea50ec376844daaf48eb5, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:50,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741866_1042 (size=87) 2024-11-09T02:19:50,827 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:19:50,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:51,157 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:51,157 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 2452fe7575fea50ec376844daaf48eb5, disabling compactions & flushes 2024-11-09T02:19:51,157 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,158 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,158 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. after waiting 0 ms 2024-11-09T02:19:51,158 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,158 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,158 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2452fe7575fea50ec376844daaf48eb5: 2024-11-09T02:19:51,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:51,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5.","families":{"info":[{"qualifier":"regioninfo","vlen":86,"tag":[],"timestamp":"1731118791159"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118791159"}]},"ts":"1731118791159"} 2024-11-09T02:19:51,161 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:51,162 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:51,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118791162"}]},"ts":"1731118791162"} 2024-11-09T02:19:51,163 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:19:51,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=2452fe7575fea50ec376844daaf48eb5, ASSIGN}] 2024-11-09T02:19:51,171 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=2452fe7575fea50ec376844daaf48eb5, ASSIGN 2024-11-09T02:19:51,172 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=2452fe7575fea50ec376844daaf48eb5, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:51,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:19:51,302 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281886 expired clientIPAndPort=172.17.0.2:60918, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:19:51,302 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281889 expired clientIPAndPort=172.17.0.2:60918, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:19:51,302 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281887 expired clientIPAndPort=172.17.0.2:60918, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:19:51,322 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=2452fe7575fea50ec376844daaf48eb5, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:51,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 2452fe7575fea50ec376844daaf48eb5, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T02:19:51,476 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:51,481 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,481 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7285): Opening region: {ENCODED => 2452fe7575fea50ec376844daaf48eb5, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:51,482 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,482 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:51,482 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7327): checking encryption for 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,482 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7330): checking classloading for 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,483 INFO [StoreOpener-2452fe7575fea50ec376844daaf48eb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,485 INFO [StoreOpener-2452fe7575fea50ec376844daaf48eb5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2452fe7575fea50ec376844daaf48eb5 columnFamilyName testFamily 2024-11-09T02:19:51,485 DEBUG [StoreOpener-2452fe7575fea50ec376844daaf48eb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:51,486 INFO [StoreOpener-2452fe7575fea50ec376844daaf48eb5-1 {}] regionserver.HStore(327): Store=2452fe7575fea50ec376844daaf48eb5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:51,487 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,488 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,491 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1085): writing seq id for 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:19:51,493 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:51,494 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1102): Opened 2452fe7575fea50ec376844daaf48eb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64745173, jitterRate=-0.03522174060344696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:51,495 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1001): Region open journal for 2452fe7575fea50ec376844daaf48eb5: 2024-11-09T02:19:51,496 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5., pid=53, masterSystemTime=1731118791476 2024-11-09T02:19:51,497 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,498 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:19:51,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=2452fe7575fea50ec376844daaf48eb5, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:51,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T02:19:51,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 2452fe7575fea50ec376844daaf48eb5, server=54f3333e9615,45997,1731118672827 in 176 msec 2024-11-09T02:19:51,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-09T02:19:51,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, region=2452fe7575fea50ec376844daaf48eb5, ASSIGN in 331 msec 2024-11-09T02:19:51,504 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:51,504 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118791504"}]},"ts":"1731118791504"} 2024-11-09T02:19:51,506 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:19:51,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:51,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync_1_-true in 1.1710 sec 2024-11-09T02:19:52,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-09T02:19:52,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:52,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:19:52,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false Metrics about Tables on a single HBase RegionServer 2024-11-09T02:19:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T02:19:53,344 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync_1_-true, procId: 51 completed 2024-11-09T02:19:53,344 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:19:53,344 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:53,348 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T02:19:53,348 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:53,348 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextExceptionAsync_1_-true assigned. 2024-11-09T02:19:53,350 DEBUG [hconnection-0x4f446b87-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:53,352 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39986, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:53,360 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:53,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T02:19:53,363 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:19:53,364 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x32b3cd9e {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:19:53,364 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x32b3cd9e {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:53,365 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40002, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:53,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true', row='', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:19:53,367 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T02:19:53,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269790 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T02:19:53,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269790 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281826 2024-11-09T02:19:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40002 deadline: 1731118796368, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:19:53,370 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269790 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:53,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T02:19:53,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269791 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T02:19:53,372 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:19:53,372 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:19:53,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269792 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281824 2024-11-09T02:19:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40002 deadline: 1731118796371, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:19:53,373 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269792 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:53,374 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:19:53,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269793 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1481684882995281823 2024-11-09T02:19:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40002 deadline: 1731118796373, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:19:53,375 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269793 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:53,376 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:19:53,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269794 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:19:53,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1481684882995281822 2024-11-09T02:19:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40002 deadline: 1731118796375, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:19:53,377 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269794 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:19:53,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:19:53,378 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a4cc413 to 127.0.0.1:57725 2024-11-09T02:19:53,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:53,379 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:19:53,379 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:19:53,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:19:53,379 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32b3cd9e to 127.0.0.1:57725 2024-11-09T02:19:53,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync[1] Thread=232 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=9901 (was 9897) - AvailableMemoryMB LEAK? - 2024-11-09T02:19:53,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=232, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=9901 2024-11-09T02:19:53,399 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5cb3e1e9 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@381140ba 2024-11-09T02:19:53,402 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5cb3e1e9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82aed42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:53,403 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d213955 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27dcff05 2024-11-09T02:19:53,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6423eebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:19:53,408 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:19:53,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:19:53,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:19:53,410 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:53,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:19:53,411 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner_1_-true" procId is: 54 2024-11-09T02:19:53,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T02:19:53,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741867_1043 (size=422) 2024-11-09T02:19:53,821 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e96580184280b371fd9cd3867da5c0c0, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:19:53,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741868_1044 (size=73) 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing e96580184280b371fd9cd3867da5c0c0, disabling compactions & flushes 2024-11-09T02:19:54,229 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. after waiting 0 ms 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,229 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for e96580184280b371fd9cd3867da5c0c0: 2024-11-09T02:19:54,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:19:54,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0.","families":{"info":[{"qualifier":"regioninfo","vlen":72,"tag":[],"timestamp":"1731118794230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118794230"}]},"ts":"1731118794230"} 2024-11-09T02:19:54,232 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:19:54,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:19:54,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118794233"}]},"ts":"1731118794233"} 2024-11-09T02:19:54,235 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:19:54,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=e96580184280b371fd9cd3867da5c0c0, ASSIGN}] 2024-11-09T02:19:54,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=e96580184280b371fd9cd3867da5c0c0, ASSIGN 2024-11-09T02:19:54,240 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=e96580184280b371fd9cd3867da5c0c0, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:19:54,391 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=e96580184280b371fd9cd3867da5c0c0, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:54,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; OpenRegionProcedure e96580184280b371fd9cd3867da5c0c0, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:19:54,404 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281880 expired clientIPAndPort=172.17.0.2:60932, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:19:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T02:19:54,545 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:19:54,549 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7285): Opening region: {ENCODED => e96580184280b371fd9cd3867da5c0c0, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:19:54,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner_1_-true e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:19:54,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7327): checking encryption for e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(7330): checking classloading for e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,551 INFO [StoreOpener-e96580184280b371fd9cd3867da5c0c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,553 INFO [StoreOpener-e96580184280b371fd9cd3867da5c0c0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e96580184280b371fd9cd3867da5c0c0 columnFamilyName testFamily 2024-11-09T02:19:54,553 DEBUG [StoreOpener-e96580184280b371fd9cd3867da5c0c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:19:54,554 INFO [StoreOpener-e96580184280b371fd9cd3867da5c0c0-1 {}] regionserver.HStore(327): Store=e96580184280b371fd9cd3867da5c0c0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:19:54,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1085): writing seq id for e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:19:54,560 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:19:54,560 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1102): Opened e96580184280b371fd9cd3867da5c0c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62678749, jitterRate=-0.06601385772228241}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:19:54,561 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegion(1001): Region open journal for e96580184280b371fd9cd3867da5c0c0: 2024-11-09T02:19:54,562 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., pid=56, masterSystemTime=1731118794545 2024-11-09T02:19:54,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,563 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=56}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:19:54,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=e96580184280b371fd9cd3867da5c0c0, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:19:54,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T02:19:54,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; OpenRegionProcedure e96580184280b371fd9cd3867da5c0c0, server=54f3333e9615,45997,1731118672827 in 173 msec 2024-11-09T02:19:54,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T02:19:54,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, region=e96580184280b371fd9cd3867da5c0c0, ASSIGN in 328 msec 2024-11-09T02:19:54,570 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:19:54,570 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118794570"}]},"ts":"1731118794570"} 2024-11-09T02:19:54,571 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:19:54,576 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=54, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:19:54,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner_1_-true in 1.1680 sec 2024-11-09T02:19:55,018 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync_1_-true' 2024-11-09T02:19:55,019 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' 2024-11-09T02:19:55,489 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:19:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-09T02:19:56,416 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner_1_-true, procId: 54 completed 2024-11-09T02:19:56,416 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:19:56,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:56,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-09T02:19:56,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:19:56,420 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-09T02:19:56,422 DEBUG [hconnection-0x5993a56a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:19:56,424 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40010, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:19:56,432 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:19:56,435 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:19:56,436 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:19:56,984 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:19:57,007 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281875 expired clientIPAndPort=172.17.0.2:60934, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:19:59,009 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281873 expired clientIPAndPort=172.17.0.2:60934, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:20:02,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:20:02,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:02,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T02:20:03,112 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281868 expired clientIPAndPort=172.17.0.2:60948, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:20:03,459 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:20:05,206 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:20:05,445 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:20:05,446 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T02:20:05.446Z, java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:285) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=9000, callDuration=11021: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms row '' on table 'hbase:testMetaScanTimeoutOnOpenScanner_1_-true' at region=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=1986ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:20:05,447 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9011 2024-11-09T02:20:05,447 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d213955 to 127.0.0.1:57725 2024-11-09T02:20:05,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:05,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:05,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:05,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:05,448 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5cb3e1e9 to 127.0.0.1:57725 2024-11-09T02:20:05,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner[1] Thread=227 (was 232), OpenFileDescriptor=446 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=9861 (was 9901) 2024-11-09T02:20:05,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=227, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=9861 2024-11-09T02:20:05,471 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69a08724 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38d7dcb0 2024-11-09T02:20:05,475 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x69a08724 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6077e2f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:05,476 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21d5d6ff to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8ad96db 2024-11-09T02:20:05,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d9b94d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:05,486 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:05,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true 2024-11-09T02:20:05,489 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:05,489 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:05,489 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync_1_-true" procId is: 57 2024-11-09T02:20:05,490 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T02:20:05,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741869_1045 (size=420) 2024-11-09T02:20:05,904 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 69b0fe0f891da7370283b06dbbed5543, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:05,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741870_1046 (size=71) 2024-11-09T02:20:05,914 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:05,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 69b0fe0f891da7370283b06dbbed5543, disabling compactions & flushes 2024-11-09T02:20:05,915 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:05,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:05,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. after waiting 0 ms 2024-11-09T02:20:05,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:05,915 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:05,915 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 69b0fe0f891da7370283b06dbbed5543: 2024-11-09T02:20:05,916 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:05,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731118805916"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118805916"}]},"ts":"1731118805916"} 2024-11-09T02:20:05,919 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:05,920 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:05,920 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118805920"}]},"ts":"1731118805920"} 2024-11-09T02:20:05,922 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:05,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=69b0fe0f891da7370283b06dbbed5543, ASSIGN}] 2024-11-09T02:20:05,929 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=69b0fe0f891da7370283b06dbbed5543, ASSIGN 2024-11-09T02:20:05,930 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=69b0fe0f891da7370283b06dbbed5543, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:06,081 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=69b0fe0f891da7370283b06dbbed5543, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:06,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 69b0fe0f891da7370283b06dbbed5543, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:06,239 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:06,249 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:06,250 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7285): Opening region: {ENCODED => 69b0fe0f891da7370283b06dbbed5543, NAME => 'hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:06,250 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync_1_-true 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,250 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:06,250 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7327): checking encryption for 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,251 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(7330): checking classloading for 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,253 INFO [StoreOpener-69b0fe0f891da7370283b06dbbed5543-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,255 INFO [StoreOpener-69b0fe0f891da7370283b06dbbed5543-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69b0fe0f891da7370283b06dbbed5543 columnFamilyName testFamily 2024-11-09T02:20:06,255 DEBUG [StoreOpener-69b0fe0f891da7370283b06dbbed5543-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:06,256 INFO [StoreOpener-69b0fe0f891da7370283b06dbbed5543-1 {}] regionserver.HStore(327): Store=69b0fe0f891da7370283b06dbbed5543/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:06,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1085): writing seq id for 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:20:06,269 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:06,270 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1102): Opened 69b0fe0f891da7370283b06dbbed5543; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60338879, jitterRate=-0.10088063776493073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:06,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegion(1001): Region open journal for 69b0fe0f891da7370283b06dbbed5543: 2024-11-09T02:20:06,272 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543., pid=59, masterSystemTime=1731118806239 2024-11-09T02:20:06,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:06,274 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=59}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:20:06,275 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=69b0fe0f891da7370283b06dbbed5543, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:06,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T02:20:06,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 69b0fe0f891da7370283b06dbbed5543, server=54f3333e9615,45997,1731118672827 in 192 msec 2024-11-09T02:20:06,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T02:20:06,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true, region=69b0fe0f891da7370283b06dbbed5543, ASSIGN in 351 msec 2024-11-09T02:20:06,281 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:06,281 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118806281"}]},"ts":"1731118806281"} 2024-11-09T02:20:06,283 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:06,286 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:06,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync_1_-true in 800 msec 2024-11-09T02:20:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T02:20:06,493 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync_1_-true, procId: 57 completed 2024-11-09T02:20:06,493 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:06,493 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:06,497 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:06,498 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:06,498 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNextAsync_1_-true assigned. 2024-11-09T02:20:06,500 DEBUG [hconnection-0x20c40030-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:06,502 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:06,512 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:06,515 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:06,515 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:20:06,517 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x69a08724 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:20:06,517 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x69a08724 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:06,518 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54566, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:06,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:20:06,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:20:06,523 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269806 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:06,523 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:20:06,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269806 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:06,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T02:20:07,021 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync_1_-true' 2024-11-09T02:20:09,960 WARN [RS-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:11,795 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:12,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-09T02:20:12,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:15,525 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9001 2024-11-09T02:20:15,525 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21d5d6ff to 127.0.0.1:57725 2024-11-09T02:20:15,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:15,526 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:15,525 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269806 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:20:15,526 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:15,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:15,526 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69a08724 to 127.0.0.1:57725 2024-11-09T02:20:15,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281810' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T02:20:15,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54566 deadline: 1731118821525, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281810' does not exist 2024-11-09T02:20:15,526 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 54f3333e9615,45997,1731118672827 for closing scanner id = -1481684882995281810 for 69b0fe0f891da7370283b06dbbed5543 of hbase:testMetaScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T02:20:15,530 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:15,531 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281855 expired clientIPAndPort=172.17.0.2:42630, userName=jenkins, regionInfo=testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:20:15,538 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync[1] Thread=231 (was 227) - Thread LEAK? -, OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9775 (was 9861) 2024-11-09T02:20:15,549 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=9775 2024-11-09T02:20:15,551 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x621bbbb0 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a8b8e07 2024-11-09T02:20:15,561 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x621bbbb0 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722c8990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:15,562 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x132a822a to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ca01a0a 2024-11-09T02:20:15,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6359ec15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:15,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T02:20:15,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:15,573 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:15,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException_1_-true" procId is: 60 2024-11-09T02:20:15,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-09T02:20:15,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741871_1047 (size=431) 2024-11-09T02:20:15,589 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 647c800ac4cc2c15506f689be4cf8b8f, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:15,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741872_1048 (size=82) 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 647c800ac4cc2c15506f689be4cf8b8f, disabling compactions & flushes 2024-11-09T02:20:15,599 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. after waiting 0 ms 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,599 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,599 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 647c800ac4cc2c15506f689be4cf8b8f: 2024-11-09T02:20:15,600 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:15,601 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731118815600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118815600"}]},"ts":"1731118815600"} 2024-11-09T02:20:15,603 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:15,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:15,604 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118815603"}]},"ts":"1731118815603"} 2024-11-09T02:20:15,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:15,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=647c800ac4cc2c15506f689be4cf8b8f, ASSIGN}] 2024-11-09T02:20:15,611 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=647c800ac4cc2c15506f689be4cf8b8f, ASSIGN 2024-11-09T02:20:15,613 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=61, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=647c800ac4cc2c15506f689be4cf8b8f, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:15,763 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=647c800ac4cc2c15506f689be4cf8b8f, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:15,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; OpenRegionProcedure 647c800ac4cc2c15506f689be4cf8b8f, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:15,918 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:15,925 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(135): Open testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,925 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7285): Opening region: {ENCODED => 647c800ac4cc2c15506f689be4cf8b8f, NAME => 'testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:15,926 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException_1_-true 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,926 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(894): Instantiated testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:15,926 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7327): checking encryption for 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,926 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(7330): checking classloading for 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,932 INFO [StoreOpener-647c800ac4cc2c15506f689be4cf8b8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,934 INFO [StoreOpener-647c800ac4cc2c15506f689be4cf8b8f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 647c800ac4cc2c15506f689be4cf8b8f columnFamilyName testFamily 2024-11-09T02:20:15,934 DEBUG [StoreOpener-647c800ac4cc2c15506f689be4cf8b8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:15,935 INFO [StoreOpener-647c800ac4cc2c15506f689be4cf8b8f-1 {}] regionserver.HStore(327): Store=647c800ac4cc2c15506f689be4cf8b8f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:15,936 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,937 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,941 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1085): writing seq id for 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:20:15,946 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:15,947 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1102): Opened 647c800ac4cc2c15506f689be4cf8b8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59177323, jitterRate=-0.11818917095661163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:15,949 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegion(1001): Region open journal for 647c800ac4cc2c15506f689be4cf8b8f: 2024-11-09T02:20:15,950 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2601): Post open deploy tasks for testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f., pid=62, masterSystemTime=1731118815918 2024-11-09T02:20:15,952 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] regionserver.HRegionServer(2628): Finished post open deploy task for testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,952 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=62}] handler.AssignRegionHandler(164): Opened testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:20:15,953 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=61 updating hbase:meta row=647c800ac4cc2c15506f689be4cf8b8f, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:15,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-09T02:20:15,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; OpenRegionProcedure 647c800ac4cc2c15506f689be4cf8b8f, server=54f3333e9615,45997,1731118672827 in 189 msec 2024-11-09T02:20:15,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T02:20:15,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, ppid=60, state=SUCCESS; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException_1_-true, region=647c800ac4cc2c15506f689be4cf8b8f, ASSIGN in 347 msec 2024-11-09T02:20:15,960 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:15,960 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118815960"}]},"ts":"1731118815960"} 2024-11-09T02:20:15,962 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testRetryOutOfOrderScannerNextException_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:15,965 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=60, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:15,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, state=SUCCESS; CreateTableProcedure table=testRetryOutOfOrderScannerNextException_1_-true in 395 msec 2024-11-09T02:20:16,025 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=60 2024-11-09T02:20:16,580 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException_1_-true, procId: 60 completed 2024-11-09T02:20:16,580 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testRetryOutOfOrderScannerNextException_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:16,581 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:16,587 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:16,587 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:16,587 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testRetryOutOfOrderScannerNextException_1_-true assigned. 2024-11-09T02:20:16,596 DEBUG [hconnection-0x3201081b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:16,598 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54798, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:16,608 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:16,610 INFO [Time-limited test {}] client.TestClientScannerTimeouts(346): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T02:20:16,614 INFO [Time-limited test {}] client.TestClientScannerTimeouts(353): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T02:20:16,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269810 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(358): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T02:20:16,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269810 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281806 2024-11-09T02:20:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54798 deadline: 1731118819617, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:20:16,618 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:20:16,622 INFO [Time-limited test {}] client.TestClientScannerTimeouts(368): Finishing scan, expecting no errors 2024-11-09T02:20:16,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269811 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(373): Testing always throw exception 2024-11-09T02:20:16,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:20:16,635 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:20:16,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269812 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=1, tableScannerId=-1481684882995281804 2024-11-09T02:20:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54798 deadline: 1731118819636, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:20:16,638 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:20:16,640 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:20:16,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269813 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=2, tableScannerId=-1481684882995281803 2024-11-09T02:20:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54798 deadline: 1731118819641, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:20:16,642 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:20:16,645 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:20:16,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269814 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:16,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(586): THROWING exception, tryNumber=3, tableScannerId=-1481684882995281802 2024-11-09T02:20:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54798 deadline: 1731118819645, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T02:20:16,646 DEBUG [Time-limited test {}] client.ScannerCallableWithReplicas(215): Scan with primary region returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:588) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:20:16,648 INFO [Time-limited test {}] client.TestClientScannerTimeouts(382): Calling scanner.next() 2024-11-09T02:20:16,649 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x132a822a to 127.0.0.1:57725 2024-11-09T02:20:16,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:16,649 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:16,650 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:16,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:16,650 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x621bbbb0 to 127.0.0.1:57725 2024-11-09T02:20:16,662 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException[1] Thread=233 (was 231) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 314), ProcessCount=11 (was 11), AvailableMemoryMB=9859 (was 9775) - AvailableMemoryMB LEAK? - 2024-11-09T02:20:16,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=9859 2024-11-09T02:20:16,674 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1233cc50 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b9a91e6 2024-11-09T02:20:16,677 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x1233cc50 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63256ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:16,678 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54a1a7f1 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@811873a 2024-11-09T02:20:16,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c9b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:16,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T02:20:16,694 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:16,694 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:16,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnRenewLease_1_-true" procId is: 63 2024-11-09T02:20:16,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T02:20:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741873_1049 (size=425) 2024-11-09T02:20:16,714 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d5566be66cd3b6f502d985d4b9c532a5, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741874_1050 (size=76) 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing d5566be66cd3b6f502d985d4b9c532a5, disabling compactions & flushes 2024-11-09T02:20:16,732 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. after waiting 0 ms 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:16,732 INFO [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:16,732 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for d5566be66cd3b6f502d985d4b9c532a5: 2024-11-09T02:20:16,733 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:16,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731118816733"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118816733"}]},"ts":"1731118816733"} 2024-11-09T02:20:16,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:16,737 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:16,737 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118816737"}]},"ts":"1731118816737"} 2024-11-09T02:20:16,739 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:16,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=d5566be66cd3b6f502d985d4b9c532a5, ASSIGN}] 2024-11-09T02:20:16,747 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=d5566be66cd3b6f502d985d4b9c532a5, ASSIGN 2024-11-09T02:20:16,748 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=d5566be66cd3b6f502d985d4b9c532a5, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:16,899 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=d5566be66cd3b6f502d985d4b9c532a5, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:16,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; OpenRegionProcedure d5566be66cd3b6f502d985d4b9c532a5, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:17,054 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:17,059 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:17,059 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7285): Opening region: {ENCODED => d5566be66cd3b6f502d985d4b9c532a5, NAME => 'testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:17,059 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnRenewLease_1_-true d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,059 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:17,060 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7327): checking encryption for d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,060 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7330): checking classloading for d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,064 INFO [StoreOpener-d5566be66cd3b6f502d985d4b9c532a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,066 INFO [StoreOpener-d5566be66cd3b6f502d985d4b9c532a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5566be66cd3b6f502d985d4b9c532a5 columnFamilyName testFamily 2024-11-09T02:20:17,066 DEBUG [StoreOpener-d5566be66cd3b6f502d985d4b9c532a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:17,067 INFO [StoreOpener-d5566be66cd3b6f502d985d4b9c532a5-1 {}] regionserver.HStore(327): Store=d5566be66cd3b6f502d985d4b9c532a5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:17,068 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,068 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,071 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1085): writing seq id for d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:20:17,074 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:17,074 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1102): Opened d5566be66cd3b6f502d985d4b9c532a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61804969, jitterRate=-0.07903419435024261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:17,076 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1001): Region open journal for d5566be66cd3b6f502d985d4b9c532a5: 2024-11-09T02:20:17,077 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5., pid=65, masterSystemTime=1731118817053 2024-11-09T02:20:17,079 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:17,079 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:20:17,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=64 updating hbase:meta row=d5566be66cd3b6f502d985d4b9c532a5, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:17,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T02:20:17,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; OpenRegionProcedure d5566be66cd3b6f502d985d4b9c532a5, server=54f3333e9615,45997,1731118672827 in 180 msec 2024-11-09T02:20:17,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-09T02:20:17,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnRenewLease_1_-true, region=d5566be66cd3b6f502d985d4b9c532a5, ASSIGN in 337 msec 2024-11-09T02:20:17,086 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:17,086 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118817086"}]},"ts":"1731118817086"} 2024-11-09T02:20:17,092 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:17,095 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:17,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnRenewLease_1_-true in 403 msec 2024-11-09T02:20:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T02:20:17,701 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnRenewLease_1_-true, procId: 63 completed 2024-11-09T02:20:17,701 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:17,701 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:17,706 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:17,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:17,706 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnRenewLease_1_-true assigned. 2024-11-09T02:20:17,708 DEBUG [hconnection-0x6f8d83f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:17,710 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54806, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:17,720 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:17,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:17,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:20:17,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 1500 2024-11-09T02:20:19,023 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException_1_-true' 2024-11-09T02:20:19,024 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnRenewLease_1_-true' 2024-11-09T02:20:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T02:20:19,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:20:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54806 deadline: 1731118820731, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:20:19,743 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:20:19,743 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269819 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:20:19,744 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 2015 2024-11-09T02:20:19,744 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54a1a7f1 to 127.0.0.1:57725 2024-11-09T02:20:19,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:19,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:19,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:19,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:19,745 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1233cc50 to 127.0.0.1:57725 2024-11-09T02:20:19,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnRenewLease[1] Thread=233 (was 233), OpenFileDescriptor=452 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 314) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9795 (was 9859) 2024-11-09T02:20:19,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=233, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=329, ProcessCount=11, AvailableMemoryMB=9795 2024-11-09T02:20:19,764 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x296a71bf to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a8753bb 2024-11-09T02:20:19,768 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x296a71bf {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b8500c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:19,769 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5377cf12 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29501aba 2024-11-09T02:20:19,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cd61a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:19,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:20:19,776 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:19,776 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:19,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner_1_-true" procId is: 66 2024-11-09T02:20:19,776 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T02:20:19,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741875_1051 (size=426) 2024-11-09T02:20:20,186 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 15f40df06f118f644828b6a5fdc8e9e9, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:20,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741876_1052 (size=77) 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 15f40df06f118f644828b6a5fdc8e9e9, disabling compactions & flushes 2024-11-09T02:20:20,594 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. after waiting 0 ms 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,594 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,594 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 15f40df06f118f644828b6a5fdc8e9e9: 2024-11-09T02:20:20,596 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:20,596 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731118820596"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118820596"}]},"ts":"1731118820596"} 2024-11-09T02:20:20,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:20,598 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:20,598 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118820598"}]},"ts":"1731118820598"} 2024-11-09T02:20:20,600 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:20,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=15f40df06f118f644828b6a5fdc8e9e9, ASSIGN}] 2024-11-09T02:20:20,604 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=15f40df06f118f644828b6a5fdc8e9e9, ASSIGN 2024-11-09T02:20:20,605 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=15f40df06f118f644828b6a5fdc8e9e9, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:20,755 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=15f40df06f118f644828b6a5fdc8e9e9, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:20,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; OpenRegionProcedure 15f40df06f118f644828b6a5fdc8e9e9, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T02:20:20,910 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:20,914 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,915 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7285): Opening region: {ENCODED => 15f40df06f118f644828b6a5fdc8e9e9, NAME => 'testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:20,915 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner_1_-true 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,915 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:20,915 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7327): checking encryption for 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,915 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(7330): checking classloading for 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,917 INFO [StoreOpener-15f40df06f118f644828b6a5fdc8e9e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,919 INFO [StoreOpener-15f40df06f118f644828b6a5fdc8e9e9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15f40df06f118f644828b6a5fdc8e9e9 columnFamilyName testFamily 2024-11-09T02:20:20,919 DEBUG [StoreOpener-15f40df06f118f644828b6a5fdc8e9e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:20,919 INFO [StoreOpener-15f40df06f118f644828b6a5fdc8e9e9-1 {}] regionserver.HStore(327): Store=15f40df06f118f644828b6a5fdc8e9e9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:20,920 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,921 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,923 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1085): writing seq id for 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:20:20,925 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:20,926 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1102): Opened 15f40df06f118f644828b6a5fdc8e9e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66715519, jitterRate=-0.005861297249794006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:20,927 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegion(1001): Region open journal for 15f40df06f118f644828b6a5fdc8e9e9: 2024-11-09T02:20:20,928 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., pid=68, masterSystemTime=1731118820909 2024-11-09T02:20:20,930 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,930 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=68}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:20:20,931 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=15f40df06f118f644828b6a5fdc8e9e9, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:20,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-09T02:20:20,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; OpenRegionProcedure 15f40df06f118f644828b6a5fdc8e9e9, server=54f3333e9615,45997,1731118672827 in 175 msec 2024-11-09T02:20:20,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T02:20:20,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true, region=15f40df06f118f644828b6a5fdc8e9e9, ASSIGN in 331 msec 2024-11-09T02:20:20,936 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:20,936 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118820936"}]},"ts":"1731118820936"} 2024-11-09T02:20:20,938 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScanner_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:20,941 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=66, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:20,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner_1_-true in 1.1670 sec 2024-11-09T02:20:21,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:20:21,427 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:22,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T02:20:22,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:22,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:20:22,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:22,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T02:20:22,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=66 2024-11-09T02:20:22,780 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner_1_-true, procId: 66 completed 2024-11-09T02:20:22,780 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:22,781 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:22,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:22,784 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:22,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScanner_1_-true assigned. 2024-11-09T02:20:22,786 DEBUG [hconnection-0xfc16f7e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:22,787 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34730, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:22,795 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:22,797 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:20:22,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:20:24,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(450): Unknown callId: 6, skipping over this response of 89 bytes 2024-11-09T02:20:24,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:20:25,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner_1_-true' 2024-11-09T02:20:25,139 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281851 expired clientIPAndPort=172.17.0.2:42644, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:20:25,805 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=5011: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=5011: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:20:25,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=4, exceptions: 2024-11-09T02:20:25.805Z, java.net.SocketTimeoutException: callTimeout=3000, callDuration=5011: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:298) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:260) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:53) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:281) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:450) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.nextWithSyncCache(ClientScanner.java:324) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:622) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScanner(TestClientScannerTimeouts.java:224) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.SocketTimeoutException: callTimeout=3000, callDuration=5011: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms row '' on table 'testNormalScanTimeoutOnOpenScanner_1_-true' at region=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=9,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:20:25,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-09T02:20:25,807 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5377cf12 to 127.0.0.1:57725 2024-11-09T02:20:25,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:25,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:25,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:25,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:25,807 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x296a71bf to 127.0.0.1:57725 2024-11-09T02:20:25,816 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner[1] Thread=231 (was 233), OpenFileDescriptor=451 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 329), ProcessCount=11 (was 11), AvailableMemoryMB=9772 (was 9795) 2024-11-09T02:20:25,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=9771 2024-11-09T02:20:25,826 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5198b778 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1383693d 2024-11-09T02:20:25,829 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5198b778 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e49bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:25,830 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x199cbc30 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28227f00 2024-11-09T02:20:25,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c952e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:25,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true 2024-11-09T02:20:25,837 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:25,837 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:25,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync_1_-true" procId is: 69 2024-11-09T02:20:25,838 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T02:20:25,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741877_1053 (size=424) 2024-11-09T02:20:26,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f0e7baf77f7cbbc7df2c5cf980de6183, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:26,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741878_1054 (size=75) 2024-11-09T02:20:26,317 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:26,417 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:26,655 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:26,656 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing f0e7baf77f7cbbc7df2c5cf980de6183, disabling compactions & flushes 2024-11-09T02:20:26,656 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,656 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,656 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. after waiting 0 ms 2024-11-09T02:20:26,656 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,656 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,656 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for f0e7baf77f7cbbc7df2c5cf980de6183: 2024-11-09T02:20:26,657 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:26,657 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183.","families":{"info":[{"qualifier":"regioninfo","vlen":74,"tag":[],"timestamp":"1731118826657"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118826657"}]},"ts":"1731118826657"} 2024-11-09T02:20:26,659 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:26,660 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:26,660 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118826660"}]},"ts":"1731118826660"} 2024-11-09T02:20:26,661 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:26,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=f0e7baf77f7cbbc7df2c5cf980de6183, ASSIGN}] 2024-11-09T02:20:26,669 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=f0e7baf77f7cbbc7df2c5cf980de6183, ASSIGN 2024-11-09T02:20:26,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=f0e7baf77f7cbbc7df2c5cf980de6183, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:26,820 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=f0e7baf77f7cbbc7df2c5cf980de6183, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:26,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; OpenRegionProcedure f0e7baf77f7cbbc7df2c5cf980de6183, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T02:20:26,974 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:26,978 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,978 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7285): Opening region: {ENCODED => f0e7baf77f7cbbc7df2c5cf980de6183, NAME => 'testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:26,979 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync_1_-true f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,979 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:26,979 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7327): checking encryption for f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,979 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7330): checking classloading for f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,981 INFO [StoreOpener-f0e7baf77f7cbbc7df2c5cf980de6183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,982 INFO [StoreOpener-f0e7baf77f7cbbc7df2c5cf980de6183-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0e7baf77f7cbbc7df2c5cf980de6183 columnFamilyName testFamily 2024-11-09T02:20:26,982 DEBUG [StoreOpener-f0e7baf77f7cbbc7df2c5cf980de6183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:26,983 INFO [StoreOpener-f0e7baf77f7cbbc7df2c5cf980de6183-1 {}] regionserver.HStore(327): Store=f0e7baf77f7cbbc7df2c5cf980de6183/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:26,984 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,984 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,986 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1085): writing seq id for f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:20:26,989 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:26,989 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1102): Opened f0e7baf77f7cbbc7df2c5cf980de6183; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69365710, jitterRate=0.03362962603569031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:26,990 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1001): Region open journal for f0e7baf77f7cbbc7df2c5cf980de6183: 2024-11-09T02:20:26,991 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183., pid=71, masterSystemTime=1731118826974 2024-11-09T02:20:26,992 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,993 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:20:26,993 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=70 updating hbase:meta row=f0e7baf77f7cbbc7df2c5cf980de6183, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:26,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T02:20:26,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, ppid=70, state=SUCCESS; OpenRegionProcedure f0e7baf77f7cbbc7df2c5cf980de6183, server=54f3333e9615,45997,1731118672827 in 172 msec 2024-11-09T02:20:26,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-09T02:20:26,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync_1_-true, region=f0e7baf77f7cbbc7df2c5cf980de6183, ASSIGN in 328 msec 2024-11-09T02:20:26,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:26,998 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118826998"}]},"ts":"1731118826998"} 2024-11-09T02:20:26,999 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNextAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:27,001 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:27,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync_1_-true in 1.1660 sec 2024-11-09T02:20:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-09T02:20:28,841 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync_1_-true, procId: 69 completed 2024-11-09T02:20:28,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNextAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:28,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:28,845 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:28,845 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:28,845 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNextAsync_1_-true assigned. 2024-11-09T02:20:28,847 DEBUG [hconnection-0x664714dd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:28,848 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34732, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:28,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:28,857 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:28,857 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:20:28,858 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5198b778 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:20:28,858 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5198b778 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:28,859 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34738, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnNextAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:20:28,861 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:20:28,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269831 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:28,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:20:28,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269831 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:28,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T02:20:31,027 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync_1_-true' 2024-11-09T02:20:31,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3003 2024-11-09T02:20:31,866 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x199cbc30 to 127.0.0.1:57725 2024-11-09T02:20:31,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:31,866 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:31,866 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269831 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:20:31,866 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:31,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:31,866 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5198b778 to 127.0.0.1:57725 2024-11-09T02:20:31,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281785' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3647) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T02:20:31,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:34738 deadline: 1731118832866, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1481684882995281785' does not exist 2024-11-09T02:20:31,872 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 54f3333e9615,45997,1731118672827 for closing scanner id = -1481684882995281785 for f0e7baf77f7cbbc7df2c5cf980de6183 of testNormalScanTimeoutOnNextAsync_1_-true failed, ignore, probably already closed org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:249) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.cleanupCalls(NettyRpcDuplexHandler.java:151) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:159) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed ... 27 more 2024-11-09T02:20:31,880 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync[1] Thread=231 (was 231), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=10879 (was 9771) - AvailableMemoryMB LEAK? - 2024-11-09T02:20:31,889 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=231, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=286, ProcessCount=11, AvailableMemoryMB=10878 2024-11-09T02:20:31,891 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59b17c65 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67a65bdd 2024-11-09T02:20:31,894 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x59b17c65 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d1f8bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:31,895 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74c9aadd to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7618553c 2024-11-09T02:20:31,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24985dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:31,901 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:31,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true 2024-11-09T02:20:31,903 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:31,903 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:31,903 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnClose_1_-true" procId is: 72 2024-11-09T02:20:31,904 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-09T02:20:31,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741879_1055 (size=416) 2024-11-09T02:20:32,316 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 2b58f24cf912d4e547dcedf3080490dd, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:32,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741880_1056 (size=67) 2024-11-09T02:20:32,331 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:32,332 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 2b58f24cf912d4e547dcedf3080490dd, disabling compactions & flushes 2024-11-09T02:20:32,332 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,332 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,332 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. after waiting 0 ms 2024-11-09T02:20:32,332 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,332 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,332 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 2b58f24cf912d4e547dcedf3080490dd: 2024-11-09T02:20:32,333 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:32,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731118832333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118832333"}]},"ts":"1731118832333"} 2024-11-09T02:20:32,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:32,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:32,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118832337"}]},"ts":"1731118832337"} 2024-11-09T02:20:32,339 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:32,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=2b58f24cf912d4e547dcedf3080490dd, ASSIGN}] 2024-11-09T02:20:32,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=2b58f24cf912d4e547dcedf3080490dd, ASSIGN 2024-11-09T02:20:32,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=2b58f24cf912d4e547dcedf3080490dd, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:32,363 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:32,481 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:32,496 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=2b58f24cf912d4e547dcedf3080490dd, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:32,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 2b58f24cf912d4e547dcedf3080490dd, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:32,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-09T02:20:32,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:32,654 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:32,660 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,660 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7285): Opening region: {ENCODED => 2b58f24cf912d4e547dcedf3080490dd, NAME => 'hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:32,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnClose_1_-true 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:32,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7327): checking encryption for 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(7330): checking classloading for 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,663 INFO [StoreOpener-2b58f24cf912d4e547dcedf3080490dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,665 INFO [StoreOpener-2b58f24cf912d4e547dcedf3080490dd-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b58f24cf912d4e547dcedf3080490dd columnFamilyName testFamily 2024-11-09T02:20:32,666 DEBUG [StoreOpener-2b58f24cf912d4e547dcedf3080490dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:32,667 INFO [StoreOpener-2b58f24cf912d4e547dcedf3080490dd-1 {}] regionserver.HStore(327): Store=2b58f24cf912d4e547dcedf3080490dd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:32,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1085): writing seq id for 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:20:32,679 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:32,680 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1102): Opened 2b58f24cf912d4e547dcedf3080490dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62447787, jitterRate=-0.06945545971393585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:32,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegion(1001): Region open journal for 2b58f24cf912d4e547dcedf3080490dd: 2024-11-09T02:20:32,682 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd., pid=74, masterSystemTime=1731118832654 2024-11-09T02:20:32,685 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=2b58f24cf912d4e547dcedf3080490dd, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:32,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,688 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=74}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:20:32,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-09T02:20:32,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 2b58f24cf912d4e547dcedf3080490dd, server=54f3333e9615,45997,1731118672827 in 191 msec 2024-11-09T02:20:32,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T02:20:32,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true, region=2b58f24cf912d4e547dcedf3080490dd, ASSIGN in 348 msec 2024-11-09T02:20:32,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:32,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118832694"}]},"ts":"1731118832694"} 2024-11-09T02:20:32,696 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:32,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:32,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnClose_1_-true in 801 msec 2024-11-09T02:20:32,746 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281845 expired clientIPAndPort=172.17.0.2:51314, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:20:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=72 2024-11-09T02:20:32,907 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnClose_1_-true, procId: 72 completed 2024-11-09T02:20:32,907 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:32,907 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:32,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32895 bytes) of info 2024-11-09T02:20:32,927 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269832 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:32,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:32,933 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:32,933 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnClose_1_-true assigned. 2024-11-09T02:20:32,936 DEBUG [hconnection-0x477f3519-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:32,937 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48522, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:32,948 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:32,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:32,958 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269835 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:20:32,958 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 6500 2024-11-09T02:20:35,497 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:20:36,271 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:20:37,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnClose_1_-true' 2024-11-09T02:20:38,162 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:39,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 6504 2024-11-09T02:20:39,459 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74c9aadd to 127.0.0.1:57725 2024-11-09T02:20:39,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:39,459 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:39,460 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:39,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:39,460 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59b17c65 to 127.0.0.1:57725 2024-11-09T02:20:39,470 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnClose[1] Thread=229 (was 231), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 286), ProcessCount=11 (was 11), AvailableMemoryMB=10548 (was 10878) 2024-11-09T02:20:39,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=229, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=10546 2024-11-09T02:20:39,483 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7eed5e87 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33746f2a 2024-11-09T02:20:39,487 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x7eed5e87 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22546747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:39,488 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cc98284 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@fc5634d 2024-11-09T02:20:39,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc81d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:39,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true 2024-11-09T02:20:39,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:39,500 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:39,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnClose_1_-true" procId is: 75 2024-11-09T02:20:39,501 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T02:20:39,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741881_1057 (size=420) 2024-11-09T02:20:39,753 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281843 expired clientIPAndPort=172.17.0.2:51314, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:20:39,913 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dafbab47240d2184eb5b053eb17e581a, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnClose_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:39,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741882_1058 (size=71) 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing dafbab47240d2184eb5b053eb17e581a, disabling compactions & flushes 2024-11-09T02:20:40,322 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. after waiting 0 ms 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,322 INFO [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,322 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnClose_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for dafbab47240d2184eb5b053eb17e581a: 2024-11-09T02:20:40,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:40,324 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731118840323"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118840323"}]},"ts":"1731118840323"} 2024-11-09T02:20:40,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:40,326 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:40,326 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118840326"}]},"ts":"1731118840326"} 2024-11-09T02:20:40,327 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:40,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dafbab47240d2184eb5b053eb17e581a, ASSIGN}] 2024-11-09T02:20:40,332 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dafbab47240d2184eb5b053eb17e581a, ASSIGN 2024-11-09T02:20:40,332 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dafbab47240d2184eb5b053eb17e581a, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:40,483 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=dafbab47240d2184eb5b053eb17e581a, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:40,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; OpenRegionProcedure dafbab47240d2184eb5b053eb17e581a, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T02:20:40,636 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:40,641 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,641 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7285): Opening region: {ENCODED => dafbab47240d2184eb5b053eb17e581a, NAME => 'testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:40,641 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnClose_1_-true dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,641 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:40,642 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7327): checking encryption for dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,642 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7330): checking classloading for dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,643 INFO [StoreOpener-dafbab47240d2184eb5b053eb17e581a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,645 INFO [StoreOpener-dafbab47240d2184eb5b053eb17e581a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dafbab47240d2184eb5b053eb17e581a columnFamilyName testFamily 2024-11-09T02:20:40,645 DEBUG [StoreOpener-dafbab47240d2184eb5b053eb17e581a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:40,645 INFO [StoreOpener-dafbab47240d2184eb5b053eb17e581a-1 {}] regionserver.HStore(327): Store=dafbab47240d2184eb5b053eb17e581a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:40,646 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,647 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,649 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1085): writing seq id for dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:20:40,651 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:40,651 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1102): Opened dafbab47240d2184eb5b053eb17e581a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65899286, jitterRate=-0.01802411675453186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:40,652 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1001): Region open journal for dafbab47240d2184eb5b053eb17e581a: 2024-11-09T02:20:40,653 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a., pid=77, masterSystemTime=1731118840636 2024-11-09T02:20:40,655 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,655 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:20:40,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=dafbab47240d2184eb5b053eb17e581a, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:40,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T02:20:40,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, ppid=76, state=SUCCESS; OpenRegionProcedure dafbab47240d2184eb5b053eb17e581a, server=54f3333e9615,45997,1731118672827 in 171 msec 2024-11-09T02:20:40,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-09T02:20:40,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnClose_1_-true, region=dafbab47240d2184eb5b053eb17e581a, ASSIGN in 327 msec 2024-11-09T02:20:40,660 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:40,660 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnClose_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118840660"}]},"ts":"1731118840660"} 2024-11-09T02:20:40,661 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnClose_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:40,663 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:40,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnClose_1_-true in 1.1650 sec 2024-11-09T02:20:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T02:20:42,504 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnClose_1_-true, procId: 75 completed 2024-11-09T02:20:42,504 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnClose_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:42,504 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:42,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32919 bytes) of info 2024-11-09T02:20:42,508 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269836 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:42,509 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnClose_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:42,510 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:42,510 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnClose_1_-true assigned. 2024-11-09T02:20:42,511 DEBUG [hconnection-0x64c510b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:42,513 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:42,520 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:42,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(464): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:42,523 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269839 number_of_rows: 0 close_scanner: true client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false 2024-11-09T02:20:42,523 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(570): SLEEPING 1500 2024-11-09T02:20:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-09T02:20:42,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:42,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-09T02:20:42,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:43,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnClose_1_-true' 2024-11-09T02:20:44,024 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 1502 2024-11-09T02:20:44,024 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1cc98284 to 127.0.0.1:57725 2024-11-09T02:20:44,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:44,024 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:44,025 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:44,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:44,025 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7eed5e87 to 127.0.0.1:57725 2024-11-09T02:20:44,034 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnClose[1] Thread=233 (was 229) - Thread LEAK? -, OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 257), ProcessCount=11 (was 11), AvailableMemoryMB=10476 (was 10546) 2024-11-09T02:20:44,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=233, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=10475 2024-11-09T02:20:44,043 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x384b6450 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c528298 2024-11-09T02:20:44,047 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x384b6450 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5ff377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:44,048 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x013266da to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42e2d266 2024-11-09T02:20:44,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e8478e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:44,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true 2024-11-09T02:20:44,055 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:44,055 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:44,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext_1_-true" procId is: 78 2024-11-09T02:20:44,056 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:44,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T02:20:44,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741883_1059 (size=419) 2024-11-09T02:20:44,465 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8e36f6e4daeb83f83d15ea0b8a72297d, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:44,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741884_1060 (size=70) 2024-11-09T02:20:44,757 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281838 expired clientIPAndPort=172.17.0.2:54780, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 8e36f6e4daeb83f83d15ea0b8a72297d, disabling compactions & flushes 2024-11-09T02:20:44,873 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. after waiting 0 ms 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:44,873 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:44,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8e36f6e4daeb83f83d15ea0b8a72297d: 2024-11-09T02:20:44,874 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:44,874 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731118844874"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118844874"}]},"ts":"1731118844874"} 2024-11-09T02:20:44,876 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:44,877 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:44,877 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118844877"}]},"ts":"1731118844877"} 2024-11-09T02:20:44,878 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:44,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=8e36f6e4daeb83f83d15ea0b8a72297d, ASSIGN}] 2024-11-09T02:20:44,884 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=8e36f6e4daeb83f83d15ea0b8a72297d, ASSIGN 2024-11-09T02:20:44,884 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=79, ppid=78, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=8e36f6e4daeb83f83d15ea0b8a72297d, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:45,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=8e36f6e4daeb83f83d15ea0b8a72297d, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:45,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; OpenRegionProcedure 8e36f6e4daeb83f83d15ea0b8a72297d, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T02:20:45,189 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:45,193 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:45,193 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7285): Opening region: {ENCODED => 8e36f6e4daeb83f83d15ea0b8a72297d, NAME => 'testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:45,194 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext_1_-true 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,194 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:45,194 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7327): checking encryption for 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,194 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7330): checking classloading for 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,196 INFO [StoreOpener-8e36f6e4daeb83f83d15ea0b8a72297d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,197 INFO [StoreOpener-8e36f6e4daeb83f83d15ea0b8a72297d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e36f6e4daeb83f83d15ea0b8a72297d columnFamilyName testFamily 2024-11-09T02:20:45,197 DEBUG [StoreOpener-8e36f6e4daeb83f83d15ea0b8a72297d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:45,198 INFO [StoreOpener-8e36f6e4daeb83f83d15ea0b8a72297d-1 {}] regionserver.HStore(327): Store=8e36f6e4daeb83f83d15ea0b8a72297d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:45,199 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,199 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,201 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1085): writing seq id for 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:20:45,203 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:45,204 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1102): Opened 8e36f6e4daeb83f83d15ea0b8a72297d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71065670, jitterRate=0.05896100401878357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:45,205 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1001): Region open journal for 8e36f6e4daeb83f83d15ea0b8a72297d: 2024-11-09T02:20:45,205 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d., pid=80, masterSystemTime=1731118845188 2024-11-09T02:20:45,207 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:45,207 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:20:45,207 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=79 updating hbase:meta row=8e36f6e4daeb83f83d15ea0b8a72297d, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:45,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-09T02:20:45,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; OpenRegionProcedure 8e36f6e4daeb83f83d15ea0b8a72297d, server=54f3333e9615,45997,1731118672827 in 172 msec 2024-11-09T02:20:45,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T02:20:45,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, ppid=78, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext_1_-true, region=8e36f6e4daeb83f83d15ea0b8a72297d, ASSIGN in 327 msec 2024-11-09T02:20:45,212 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:45,212 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118845212"}]},"ts":"1731118845212"} 2024-11-09T02:20:45,213 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:45,216 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=78, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:45,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnNext_1_-true in 1.1630 sec 2024-11-09T02:20:46,143 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=78 2024-11-09T02:20:47,059 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext_1_-true, procId: 78 completed 2024-11-09T02:20:47,060 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:47,060 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:47,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32893 bytes) of info 2024-11-09T02:20:47,064 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269840 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:47,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:47,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:47,066 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnNext_1_-true assigned. 2024-11-09T02:20:47,067 DEBUG [hconnection-0x1764496a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:47,069 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35706, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:47,075 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:47,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:47,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:20:47,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269843 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:47,080 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:20:47,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269843 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:47,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 3500 2024-11-09T02:20:48,861 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281833 expired clientIPAndPort=172.17.0.2:48414, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:20:49,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext_1_-true' 2024-11-09T02:20:50,085 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=3000, callDuration=4010: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=3000, callDuration=4010: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms row '' on table 'testNormalScanTimeoutOnNext_1_-true' at region=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=3004ms, rpcTimeout=3000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:20:50,086 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 3000, and got 3005 2024-11-09T02:20:50,086 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x013266da to 127.0.0.1:57725 2024-11-09T02:20:50,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:50,086 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:20:50,086 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:20:50,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:20:50,086 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x384b6450 to 127.0.0.1:57725 2024-11-09T02:20:50,096 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext[1] Thread=231 (was 233), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=10428 (was 10475) 2024-11-09T02:20:50,104 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=231, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=10428 2024-11-09T02:20:50,106 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f0dde7c to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61728c6d 2024-11-09T02:20:50,109 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x6f0dde7c {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50655267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:50,110 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63c0dc13 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@643be889 2024-11-09T02:20:50,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6544344c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:20:50,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:20:50,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T02:20:50,117 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:20:50,118 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:50,118 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnRenewLease_1_-true" procId is: 81 2024-11-09T02:20:50,118 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:20:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T02:20:50,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741885_1061 (size=421) 2024-11-09T02:20:50,533 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 37354a7c4ac00e13f36e6363dec270fb, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnRenewLease_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:20:50,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741886_1062 (size=72) 2024-11-09T02:20:50,582 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:20:50,862 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281831 expired clientIPAndPort=172.17.0.2:48414, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 37354a7c4ac00e13f36e6363dec270fb, disabling compactions & flushes 2024-11-09T02:20:50,942 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. after waiting 0 ms 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:50,942 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:50,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnRenewLease_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 37354a7c4ac00e13f36e6363dec270fb: 2024-11-09T02:20:50,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:20:50,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731118850943"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118850943"}]},"ts":"1731118850943"} 2024-11-09T02:20:50,945 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:20:50,945 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:20:50,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118850945"}]},"ts":"1731118850945"} 2024-11-09T02:20:50,947 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:20:50,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=37354a7c4ac00e13f36e6363dec270fb, ASSIGN}] 2024-11-09T02:20:50,952 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=37354a7c4ac00e13f36e6363dec270fb, ASSIGN 2024-11-09T02:20:50,952 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=37354a7c4ac00e13f36e6363dec270fb, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:20:51,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=37354a7c4ac00e13f36e6363dec270fb, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:51,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 37354a7c4ac00e13f36e6363dec270fb, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:20:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T02:20:51,256 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:20:51,261 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:51,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7285): Opening region: {ENCODED => 37354a7c4ac00e13f36e6363dec270fb, NAME => 'hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:20:51,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnRenewLease_1_-true 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:20:51,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7327): checking encryption for 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(7330): checking classloading for 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,263 INFO [StoreOpener-37354a7c4ac00e13f36e6363dec270fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,265 INFO [StoreOpener-37354a7c4ac00e13f36e6363dec270fb-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37354a7c4ac00e13f36e6363dec270fb columnFamilyName testFamily 2024-11-09T02:20:51,265 DEBUG [StoreOpener-37354a7c4ac00e13f36e6363dec270fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:20:51,265 INFO [StoreOpener-37354a7c4ac00e13f36e6363dec270fb-1 {}] regionserver.HStore(327): Store=37354a7c4ac00e13f36e6363dec270fb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:20:51,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:20:51,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1085): writing seq id for 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:20:51,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:20:51,271 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1102): Opened 37354a7c4ac00e13f36e6363dec270fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64336580, jitterRate=-0.041310250759124756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:20:51,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegion(1001): Region open journal for 37354a7c4ac00e13f36e6363dec270fb: 2024-11-09T02:20:51,273 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb., pid=83, masterSystemTime=1731118851256 2024-11-09T02:20:51,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:51,274 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=83}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:20:51,275 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=37354a7c4ac00e13f36e6363dec270fb, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:20:51,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T02:20:51,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 37354a7c4ac00e13f36e6363dec270fb, server=54f3333e9615,45997,1731118672827 in 171 msec 2024-11-09T02:20:51,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-09T02:20:51,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true, region=37354a7c4ac00e13f36e6363dec270fb, ASSIGN in 326 msec 2024-11-09T02:20:51,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:20:51,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnRenewLease_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118851279"}]},"ts":"1731118851279"} 2024-11-09T02:20:51,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnRenewLease_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:20:51,283 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:20:51,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnRenewLease_1_-true in 1.1670 sec 2024-11-09T02:20:52,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true 2024-11-09T02:20:52,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:52,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T02:20:52,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:20:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T02:20:53,122 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnRenewLease_1_-true, procId: 81 completed 2024-11-09T02:20:53,122 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnRenewLease_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:20:53,122 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:53,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32784 bytes) of info 2024-11-09T02:20:53,127 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269844 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:20:53,128 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned to meta. Checking AM states. 2024-11-09T02:20:53,129 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:20:53,129 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnRenewLease_1_-true assigned. 2024-11-09T02:20:53,131 DEBUG [hconnection-0x1c5e3eef-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:20:53,132 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58182, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:20:53,140 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:20:53,142 INFO [Time-limited test {}] client.TestClientScannerTimeouts(451): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:20:53,144 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:20:53,145 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 6500 2024-11-09T02:20:55,033 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnRenewLease_1_-true' 2024-11-09T02:20:56,808 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:20:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(450): Unknown callId: 7, skipping over this response of 12 bytes 2024-11-09T02:21:00,159 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:21:00,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Scan size: 45 connection: 172.17.0.2:58182 deadline: 1731118862145, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true 2024-11-09T02:21:00,160 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException): org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2024-11-09T02:21:00,160 DEBUG [Time-limited test {}] client.ClientScanner(606): scanner failed to renew lease org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor60.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ScannerCallable.next(ScannerCallable.java:199) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:263) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallable.rpcCall(ScannerCallable.java:62) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithoutRetries(RpcRetryingCallerImpl.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:437) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:405) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException: Expected nextCallSeq: 1 But the nextCallSeq got from client: 0; request=scanner_id: 16965059190714269847 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: true at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkScanNextCallSeq(RSRpcServices.java:3277) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3655) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:563) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-09T02:21:00,161 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 7018 2024-11-09T02:21:00,161 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63c0dc13 to 127.0.0.1:57725 2024-11-09T02:21:00,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:00,161 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:21:00,161 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:21:00,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:00,162 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f0dde7c to 127.0.0.1:57725 2024-11-09T02:21:00,170 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnRenewLease[1] Thread=230 (was 231), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=191 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=10219 (was 10428) 2024-11-09T02:21:00,179 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=230, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=191, ProcessCount=11, AvailableMemoryMB=10218 2024-11-09T02:21:00,180 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19738ced to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@615bc1aa 2024-11-09T02:21:00,183 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x19738ced {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4173c3f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:00,184 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x796d70bb to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@773f54fa 2024-11-09T02:21:00,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722eaa14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:00,189 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:00,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:00,191 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:21:00,191 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:00,191 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync_1_-true" procId is: 84 2024-11-09T02:21:00,191 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:21:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T02:21:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741887_1063 (size=427) 2024-11-09T02:21:00,601 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c899c96d16d3f8a690a93440304a00e0, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:21:00,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741888_1064 (size=78) 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing c899c96d16d3f8a690a93440304a00e0, disabling compactions & flushes 2024-11-09T02:21:01,008 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. after waiting 0 ms 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,008 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,008 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for c899c96d16d3f8a690a93440304a00e0: 2024-11-09T02:21:01,009 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:21:01,009 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731118861009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118861009"}]},"ts":"1731118861009"} 2024-11-09T02:21:01,011 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:21:01,011 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:21:01,011 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118861011"}]},"ts":"1731118861011"} 2024-11-09T02:21:01,012 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:21:01,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c899c96d16d3f8a690a93440304a00e0, ASSIGN}] 2024-11-09T02:21:01,017 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c899c96d16d3f8a690a93440304a00e0, ASSIGN 2024-11-09T02:21:01,018 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=85, ppid=84, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c899c96d16d3f8a690a93440304a00e0, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:21:01,168 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c899c96d16d3f8a690a93440304a00e0, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:01,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; OpenRegionProcedure c899c96d16d3f8a690a93440304a00e0, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:21:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T02:21:01,322 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:21:01,329 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7285): Opening region: {ENCODED => c899c96d16d3f8a690a93440304a00e0, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:01,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync_1_-true c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:01,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7327): checking encryption for c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,330 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(7330): checking classloading for c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,332 INFO [StoreOpener-c899c96d16d3f8a690a93440304a00e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,333 INFO [StoreOpener-c899c96d16d3f8a690a93440304a00e0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c899c96d16d3f8a690a93440304a00e0 columnFamilyName testFamily 2024-11-09T02:21:01,333 DEBUG [StoreOpener-c899c96d16d3f8a690a93440304a00e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:01,334 INFO [StoreOpener-c899c96d16d3f8a690a93440304a00e0-1 {}] regionserver.HStore(327): Store=c899c96d16d3f8a690a93440304a00e0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:01,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,336 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,338 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1085): writing seq id for c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:01,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:01,340 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1102): Opened c899c96d16d3f8a690a93440304a00e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60005015, jitterRate=-0.10585559904575348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:21:01,341 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegion(1001): Region open journal for c899c96d16d3f8a690a93440304a00e0: 2024-11-09T02:21:01,342 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., pid=86, masterSystemTime=1731118861322 2024-11-09T02:21:01,343 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,343 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=86}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:01,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=85 updating hbase:meta row=c899c96d16d3f8a690a93440304a00e0, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:01,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-09T02:21:01,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; OpenRegionProcedure c899c96d16d3f8a690a93440304a00e0, server=54f3333e9615,45997,1731118672827 in 175 msec 2024-11-09T02:21:01,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T02:21:01,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, ppid=84, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, region=c899c96d16d3f8a690a93440304a00e0, ASSIGN in 330 msec 2024-11-09T02:21:01,348 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:21:01,348 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118861348"}]},"ts":"1731118861348"} 2024-11-09T02:21:01,350 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:21:01,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=84, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:21:01,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true in 1.1630 sec 2024-11-09T02:21:02,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:02,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:02,975 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281817 expired clientIPAndPort=172.17.0.2:40010, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=84 2024-11-09T02:21:03,194 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true, procId: 84 completed 2024-11-09T02:21:03,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:21:03,195 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:03,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32878 bytes) of info 2024-11-09T02:21:03,198 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269849 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:21:03,199 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T02:21:03,200 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:03,200 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-09T02:21:03,201 DEBUG [hconnection-0x237ba6f0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:03,202 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:03,209 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:21:03,211 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:21:03,211 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:21:03,212 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x19738ced {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:21:03,212 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x19738ced {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:03,214 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33212, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:21:03,215 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:21:06,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:07,034 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-09T02:21:09,225 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-09T02:21:09,225 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-09T02:21:09,225 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:21:09,506 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:09,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T02:21:09,980 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281815 expired clientIPAndPort=172.17.0.2:40010, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:21:10,235 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 6500 2024-11-09T02:21:10,653 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:40811,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:12,215 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1980ms, rpcTimeout=1976ms 2024-11-09T02:21:12,215 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1980ms, rpcTimeout=1976ms 2024-11-09T02:21:12,215 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:21:12,216 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T02:21:09.225Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=6010ms, rpcTimeout=6000ms 2024-11-09T02:21:12.215Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1980ms, rpcTimeout=1976ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testMetaScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:294) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1980ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=1980ms, rpcTimeout=1976ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T02:21:12,217 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 6000, and got 9006 2024-11-09T02:21:12,217 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x796d70bb to 127.0.0.1:57725 2024-11-09T02:21:12,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:12,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:21:12,218 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:21:12,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:12,218 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x19738ced to 127.0.0.1:57725 2024-11-09T02:21:12,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync[1] Thread=225 (was 230), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 191), ProcessCount=11 (was 11), AvailableMemoryMB=10098 (was 10218) 2024-11-09T02:21:12,237 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=225, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=10098 2024-11-09T02:21:12,239 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22aba2d3 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76bb4593 2024-11-09T02:21:12,245 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x22aba2d3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb5b7f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:12,246 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x014686e1 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7268f750 2024-11-09T02:21:12,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2a6ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:12,251 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:12,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true 2024-11-09T02:21:12,253 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:21:12,253 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:12,253 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext_1_-true" procId is: 87 2024-11-09T02:21:12,254 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:21:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T02:21:12,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741889_1065 (size=415) 2024-11-09T02:21:12,667 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 65c9e222499018697aa1704e3adfc51e, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:21:12,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741890_1066 (size=66) 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 65c9e222499018697aa1704e3adfc51e, disabling compactions & flushes 2024-11-09T02:21:13,075 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. after waiting 0 ms 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,075 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,075 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 65c9e222499018697aa1704e3adfc51e: 2024-11-09T02:21:13,076 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:21:13,076 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e.","families":{"info":[{"qualifier":"regioninfo","vlen":65,"tag":[],"timestamp":"1731118873076"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118873076"}]},"ts":"1731118873076"} 2024-11-09T02:21:13,077 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:21:13,078 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:21:13,078 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118873078"}]},"ts":"1731118873078"} 2024-11-09T02:21:13,079 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:21:13,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=65c9e222499018697aa1704e3adfc51e, ASSIGN}] 2024-11-09T02:21:13,084 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=65c9e222499018697aa1704e3adfc51e, ASSIGN 2024-11-09T02:21:13,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=65c9e222499018697aa1704e3adfc51e, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:21:13,235 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=65c9e222499018697aa1704e3adfc51e, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:13,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure 65c9e222499018697aa1704e3adfc51e, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:21:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T02:21:13,389 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:21:13,394 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(135): Open hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,394 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7285): Opening region: {ENCODED => 65c9e222499018697aa1704e3adfc51e, NAME => 'hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:13,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext_1_-true 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(894): Instantiated hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:13,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7327): checking encryption for 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(7330): checking classloading for 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,396 INFO [StoreOpener-65c9e222499018697aa1704e3adfc51e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,398 INFO [StoreOpener-65c9e222499018697aa1704e3adfc51e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65c9e222499018697aa1704e3adfc51e columnFamilyName testFamily 2024-11-09T02:21:13,398 DEBUG [StoreOpener-65c9e222499018697aa1704e3adfc51e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:13,398 INFO [StoreOpener-65c9e222499018697aa1704e3adfc51e-1 {}] regionserver.HStore(327): Store=65c9e222499018697aa1704e3adfc51e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:13,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1085): writing seq id for 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:13,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:13,404 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1102): Opened 65c9e222499018697aa1704e3adfc51e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64965826, jitterRate=-0.031933754682540894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:21:13,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegion(1001): Region open journal for 65c9e222499018697aa1704e3adfc51e: 2024-11-09T02:21:13,406 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e., pid=89, masterSystemTime=1731118873389 2024-11-09T02:21:13,407 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,407 INFO [RS_OPEN_PRIORITY_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=89}] handler.AssignRegionHandler(164): Opened hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:13,408 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=65c9e222499018697aa1704e3adfc51e, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:13,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T02:21:13,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure 65c9e222499018697aa1704e3adfc51e, server=54f3333e9615,45997,1731118672827 in 172 msec 2024-11-09T02:21:13,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T02:21:13,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true, region=65c9e222499018697aa1704e3adfc51e, ASSIGN in 327 msec 2024-11-09T02:21:13,412 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:21:13,412 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118873412"}]},"ts":"1731118873412"} 2024-11-09T02:21:13,413 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:testMetaScanTimeoutOnNext_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:21:13,415 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:21:13,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext_1_-true in 1.1640 sec 2024-11-09T02:21:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T02:21:15,257 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext_1_-true, procId: 87 completed 2024-11-09T02:21:15,257 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:21:15,257 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:15,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32830 bytes) of info 2024-11-09T02:21:15,261 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269856 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:21:15,262 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned to meta. Checking AM states. 2024-11-09T02:21:15,262 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:15,262 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table hbase:testMetaScanTimeoutOnNext_1_-true assigned. 2024-11-09T02:21:15,264 DEBUG [hconnection-0x572005c7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:15,265 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:15,271 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:21:15,273 INFO [Time-limited test {}] client.TestClientScannerTimeouts(407): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T02:21:15,274 INFO [Time-limited test {}] client.TestClientScannerTimeouts(413): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T02:21:15,274 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269859 number_of_rows: 1 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:21:15,275 INFO [Time-limited test {}] client.TestClientScannerTimeouts(417): Making second next() RPC, expecting timeout 2024-11-09T02:21:15,276 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269859 number_of_rows: 1 close_scanner: false next_call_seq: 1 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:21:15,276 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(596): SLEEPING 9500 2024-11-09T02:21:16,085 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281810 expired clientIPAndPort=172.17.0.2:54566, userName=jenkins, regionInfo=hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:16,686 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281806 expired clientIPAndPort=172.17.0.2:54798, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:16,686 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281804 expired clientIPAndPort=172.17.0.2:54798, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:16,686 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281803 expired clientIPAndPort=172.17.0.2:54798, userName=jenkins, regionInfo=testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:16,736 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:21:18,929 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:19,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext_1_-true' 2024-11-09T02:21:19,789 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281797 expired clientIPAndPort=172.17.0.2:54806, userName=jenkins, regionInfo=testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:21,266 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T02:21:22,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-09T02:21:22,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:24,285 DEBUG [Time-limited test {}] client.ResultBoundedCompletionService(217): Replica 0 returns java.net.SocketTimeoutException: callTimeout=9000, callDuration=10014: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 java.net.SocketTimeoutException: callTimeout=9000, callDuration=10014: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms row '' on table 'hbase:testMetaScanTimeoutOnNext_1_-true' at region=hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e., hostname=54f3333e9615,45997,1731118672827, seqNum=2 at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:172) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:74) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ... 1 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=8,methodName=Scan], waitTime=9010ms, rpcTimeout=9000ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ... 6 more 2024-11-09T02:21:24,285 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 9000, and got 9010 2024-11-09T02:21:24,285 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x014686e1 to 127.0.0.1:57725 2024-11-09T02:21:24,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:24,286 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:21:24,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:21:24,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:24,286 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22aba2d3 to 127.0.0.1:57725 2024-11-09T02:21:24,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext[1] Thread=227 (was 225) - Thread LEAK? -, OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 162), ProcessCount=11 (was 11), AvailableMemoryMB=10009 (was 10098) 2024-11-09T02:21:24,302 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=227, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=126, ProcessCount=11, AvailableMemoryMB=10009 2024-11-09T02:21:24,304 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5795d494 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7fc502cd 2024-11-09T02:21:24,307 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5795d494 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee43c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:24,308 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63a97306 to 127.0.0.1:57725 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c75e020 2024-11-09T02:21:24,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453da24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:21:24,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:21:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] procedure2.ProcedureExecutor(1098): Stored pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:24,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:21:24,314 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:24,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync_1_-true" procId is: 90 2024-11-09T02:21:24,315 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:21:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T02:21:24,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741891_1067 (size=431) 2024-11-09T02:21:24,392 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281792 expired clientIPAndPort=172.17.0.2:34730, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:24,724 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 222cbbc69904e90550664cf8da7b9c60, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync_1_-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277 2024-11-09T02:21:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741892_1068 (size=82) 2024-11-09T02:21:24,776 WARN [RS-EventLoopGroup-3-1 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1681): Closing 222cbbc69904e90550664cf8da7b9c60, disabling compactions & flushes 2024-11-09T02:21:25,131 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. after waiting 0 ms 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,131 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,131 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync_1_-true-pool-0 {}] regionserver.HRegion(1635): Region close journal for 222cbbc69904e90550664cf8da7b9c60: 2024-11-09T02:21:25,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:21:25,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60.","families":{"info":[{"qualifier":"regioninfo","vlen":81,"tag":[],"timestamp":"1731118885132"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118885132"}]},"ts":"1731118885132"} 2024-11-09T02:21:25,134 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T02:21:25,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:21:25,135 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118885135"}]},"ts":"1731118885135"} 2024-11-09T02:21:25,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLING in hbase:meta 2024-11-09T02:21:25,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=222cbbc69904e90550664cf8da7b9c60, ASSIGN}] 2024-11-09T02:21:25,140 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=222cbbc69904e90550664cf8da7b9c60, ASSIGN 2024-11-09T02:21:25,141 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=222cbbc69904e90550664cf8da7b9c60, ASSIGN; state=OFFLINE, location=54f3333e9615,45997,1731118672827; forceNewPlan=false, retain=false 2024-11-09T02:21:25,291 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=222cbbc69904e90550664cf8da7b9c60, regionState=OPENING, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:25,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; OpenRegionProcedure 222cbbc69904e90550664cf8da7b9c60, server=54f3333e9615,45997,1731118672827}] 2024-11-09T02:21:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T02:21:25,445 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 54f3333e9615,45997,1731118672827 2024-11-09T02:21:25,451 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(135): Open testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,451 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7285): Opening region: {ENCODED => 222cbbc69904e90550664cf8da7b9c60, NAME => 'testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:21:25,451 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync_1_-true 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,451 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(894): Instantiated testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:21:25,451 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7327): checking encryption for 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,451 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7330): checking classloading for 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,453 INFO [StoreOpener-222cbbc69904e90550664cf8da7b9c60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,454 INFO [StoreOpener-222cbbc69904e90550664cf8da7b9c60-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 222cbbc69904e90550664cf8da7b9c60 columnFamilyName testFamily 2024-11-09T02:21:25,455 DEBUG [StoreOpener-222cbbc69904e90550664cf8da7b9c60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:21:25,455 INFO [StoreOpener-222cbbc69904e90550664cf8da7b9c60-1 {}] regionserver.HStore(327): Store=222cbbc69904e90550664cf8da7b9c60/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:21:25,456 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,457 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,458 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1085): writing seq id for 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:25,461 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:21:25,461 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1102): Opened 222cbbc69904e90550664cf8da7b9c60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60977569, jitterRate=-0.09136341512203217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:21:25,462 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1001): Region open journal for 222cbbc69904e90550664cf8da7b9c60: 2024-11-09T02:21:25,463 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2601): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., pid=92, masterSystemTime=1731118885445 2024-11-09T02:21:25,464 DEBUG [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2628): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,465 INFO [RS_OPEN_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(164): Opened testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:25,465 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=222cbbc69904e90550664cf8da7b9c60, regionState=OPEN, openSeqNum=2, regionLocation=54f3333e9615,45997,1731118672827 2024-11-09T02:21:25,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T02:21:25,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; OpenRegionProcedure 222cbbc69904e90550664cf8da7b9c60, server=54f3333e9615,45997,1731118672827 in 173 msec 2024-11-09T02:21:25,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-09T02:21:25,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true, region=222cbbc69904e90550664cf8da7b9c60, ASSIGN in 328 msec 2024-11-09T02:21:25,470 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:21:25,470 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync_1_-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118885470"}]},"ts":"1731118885470"} 2024-11-09T02:21:25,471 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync_1_-true, state=ENABLED in hbase:meta 2024-11-09T02:21:25,474 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=90, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:21:25,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, state=SUCCESS; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync_1_-true in 1.1610 sec 2024-11-09T02:21:26,394 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281790 expired clientIPAndPort=172.17.0.2:34730, userName=jenkins, regionInfo=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40839 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=90 2024-11-09T02:21:27,318 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync_1_-true, procId: 90 completed 2024-11-09T02:21:27,318 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(3531): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync_1_-true get assigned. Timeout = 60000ms 2024-11-09T02:21:27,319 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:27,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45997 {}] regionserver.StoreScanner(1133): Switch to stream read (scanned=32890 bytes) of info 2024-11-09T02:21:27,323 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(562): Got request scanner_id: 16965059190714269860 number_of_rows: 2147483647 close_scanner: false next_call_seq: 0 client_handles_partials: true client_handles_heartbeats: true track_scan_metrics: false renew: false 2024-11-09T02:21:27,324 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3585): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned to meta. Checking AM states. 2024-11-09T02:21:27,324 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:21:27,324 INFO [Time-limited test {}] hbase.HBaseTestingUtility(3605): All regions for table testNormalScanTimeoutOnOpenScannerAsync_1_-true assigned. 2024-11-09T02:21:27,327 DEBUG [hconnection-0x5b39c879-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:27,328 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33928, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:27,336 INFO [Time-limited test {}] client.TestClientScannerTimeouts(161): Wrote our four values 2024-11-09T02:21:27,338 INFO [Time-limited test {}] client.TestClientScannerTimeouts(433): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T02:21:27,338 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching meta region location from registry 2024-11-09T02:21:27,339 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5795d494 {}] client.ConnectionUtils(581): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54f3333e9615,45997,1731118672827, seqNum=-1] 2024-11-09T02:21:27,339 DEBUG [ReadOnlyZKClient-127.0.0.1:57725@0x5795d494 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:21:27,340 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33938, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:21:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:21:27,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:21:28,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-09T02:21:28,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-09T02:21:28,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:21:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(450): Unknown callId: 1, skipping over this response of 89 bytes 2024-11-09T02:21:29,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(311): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync_1_-true', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2] 2024-11-09T02:21:29,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45997 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(611): openScanner SLEEPING 1500 2024-11-09T02:21:30,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=982ms 2024-11-09T02:21:30,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=982ms 2024-11-09T02:21:30,345 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60., hostname=54f3333e9615,45997,1731118672827, seqNum=2 from cache 2024-11-09T02:21:30,346 INFO [Time-limited test {}] client.TestClientScannerTimeouts(439): Got error org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2024-11-09T02:21:28.345Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=1,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-11-09T02:21:30.345Z, org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.expectTimeoutOnOpenScanner(TestClientScannerTimeouts.java:436) ~[test-classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.testNormalScanTimeoutOnOpenScannerAsync(TestClientScannerTimeouts.java:233) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.tryScheduleRetry(AsyncRpcRetryingCaller.java:130) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:206) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncClientScanner.lambda$callOpenScanner$0(AsyncClientScanner.java:181) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setTimeout(Call.java:109) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:171) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=54f3333e9615:45997 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:253) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 11 more Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=3,methodName=Scan], waitTime=989ms, rpcTimeout=982ms at org.apache.hadoop.hbase.ipc.RpcConnection$1.run(RpcConnection.java:172) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 6 more 2024-11-09T02:21:30,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(477): Expected duration >= 1000, and got 3009 2024-11-09T02:21:30,347 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63a97306 to 127.0.0.1:57725 2024-11-09T02:21:30,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:30,347 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-09T02:21:30,347 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.after(TestClientScannerTimeouts.java:135) at jdk.internal.reflect.GeneratedMethodAccessor162.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:21:30,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:30,348 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5795d494 to 127.0.0.1:57725 2024-11-09T02:21:30,357 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync[1] Thread=231 (was 227) - Thread LEAK? -, OpenFileDescriptor=448 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=124 (was 126), ProcessCount=11 (was 11), AvailableMemoryMB=9996 (was 10009) 2024-11-09T02:21:30,357 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-09T02:21:30,357 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T02:21:30,357 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x574a453f to 127.0.0.1:57725 2024-11-09T02:21:30,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:30,357 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T02:21:30,358 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1469822454, stopped=false 2024-11-09T02:21:30,358 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=54f3333e9615,40839,1731118672115 2024-11-09T02:21:30,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:21:30,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:21:30,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-09T02:21:30,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:30,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:30,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:30,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:21:30,360 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '54f3333e9615,45997,1731118672827' ***** 2024-11-09T02:21:30,360 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-09T02:21:30,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:21:30,361 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T02:21:30,361 INFO [RS:0;54f3333e9615:45997 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T02:21:30,361 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-09T02:21:30,361 INFO [RS:0;54f3333e9615:45997 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T02:21:30,361 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for c7cadccf0db1443c03afbf4a8ffb7c74 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 970ccc55dc0014f96f14429b0893f112 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 2452fe7575fea50ec376844daaf48eb5 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 2ac227a2fa5b5a4d941fcbac940cb5fb 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 92e0494411be1d7676e60e07472411c6 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 27cb43c1981e47632a87e1850b149c1e 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for fd148266e0f7adabaf27b8fe8e091c94 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for c899c96d16d3f8a690a93440304a00e0 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 755bcccae28504c223e008c40b24c6c4 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 37354a7c4ac00e13f36e6363dec270fb 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 6235bfc49272e5114ec5c10328305fa2 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 222cbbc69904e90550664cf8da7b9c60 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 65c9e222499018697aa1704e3adfc51e 2024-11-09T02:21:30,362 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c7cadccf0db1443c03afbf4a8ffb7c74, disabling compactions & flushes 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 1e9b57e787ebaa31f7960f0e0be0fb6f 2024-11-09T02:21:30,362 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 3152c4d9a734cd45a64adc4eaba61986 2024-11-09T02:21:30,362 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 04a6b35baf34def60d1b7d01a8823d80 2024-11-09T02:21:30,362 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. after waiting 0 ms 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for ce03aa7d3b74ad9aaf2d966daa46e587 2024-11-09T02:21:30,362 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 69b0fe0f891da7370283b06dbbed5543 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for ac58767bfb05dc96875b60142d320308 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 15f40df06f118f644828b6a5fdc8e9e9 2024-11-09T02:21:30,362 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for dafbab47240d2184eb5b053eb17e581a 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 647c800ac4cc2c15506f689be4cf8b8f 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 8efa319d73410a0150970c2107175f31 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 2b58f24cf912d4e547dcedf3080490dd 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1224): stopping server 54f3333e9615,45997,1731118672827 2024-11-09T02:21:30,363 DEBUG [RS:0;54f3333e9615:45997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T02:21:30,363 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c7cadccf0db1443c03afbf4a8ffb7c74 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-09T02:21:30,363 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 30 regions to close 2024-11-09T02:21:30,363 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:21:30,364 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T02:21:30,364 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T02:21:30,364 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:21:30,364 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:21:30,364 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=47.95 KB heapSize=74.34 KB 2024-11-09T02:21:30,364 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 1588230740, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, 970ccc55dc0014f96f14429b0893f112, ac58767bfb05dc96875b60142d320308, c7cadccf0db1443c03afbf4a8ffb7c74, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:30,417 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/.tmp/testFamily/96f7029f597141d6bf5282f1873f3de7 is 61, key is row-0/testFamily:testQualifier/1731118707823/Put/seqid=0 2024-11-09T02:21:30,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741893_1069 (size=5049) 2024-11-09T02:21:30,432 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/info/eaaa06c615f44200bce4fa13b7cd2f59 is 227, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74./info:regioninfo/1731118707613/Put/seqid=0 2024-11-09T02:21:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741894_1070 (size=51873) 2024-11-09T02:21:30,564 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 1588230740, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, 970ccc55dc0014f96f14429b0893f112, ac58767bfb05dc96875b60142d320308, c7cadccf0db1443c03afbf4a8ffb7c74, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:30,680 INFO [regionserver/54f3333e9615:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T02:21:30,680 INFO [regionserver/54f3333e9615:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T02:21:30,765 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 1588230740, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, 970ccc55dc0014f96f14429b0893f112, ac58767bfb05dc96875b60142d320308, c7cadccf0db1443c03afbf4a8ffb7c74, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:30,829 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/.tmp/testFamily/96f7029f597141d6bf5282f1873f3de7 2024-11-09T02:21:30,837 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.72 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/info/eaaa06c615f44200bce4fa13b7cd2f59 2024-11-09T02:21:30,857 WARN [RS-EventLoopGroup-3-2 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T02:21:30,876 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/.tmp/testFamily/96f7029f597141d6bf5282f1873f3de7 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/testFamily/96f7029f597141d6bf5282f1873f3de7 2024-11-09T02:21:30,883 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/testFamily/96f7029f597141d6bf5282f1873f3de7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:30,886 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c7cadccf0db1443c03afbf4a8ffb7c74 in 523ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:30,893 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_0_-false/c7cadccf0db1443c03afbf4a8ffb7c74/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:30,896 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c7cadccf0db1443c03afbf4a8ffb7c74: 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_0_-false,,1731118706772.c7cadccf0db1443c03afbf4a8ffb7c74. 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 970ccc55dc0014f96f14429b0893f112, disabling compactions & flushes 2024-11-09T02:21:30,896 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. after waiting 0 ms 2024-11-09T02:21:30,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:21:30,897 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 970ccc55dc0014f96f14429b0893f112 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:30,899 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/table/44e8388d183945a1b8bed4e53f90cb57 is 89, key is testRetryOutOfOrderScannerNextExceptionAsync_0_-false/table:state/1731118707650/Put/seqid=0 2024-11-09T02:21:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741895_1071 (size=9821) 2024-11-09T02:21:30,910 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.23 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/table/44e8388d183945a1b8bed4e53f90cb57 2024-11-09T02:21:30,919 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/info/eaaa06c615f44200bce4fa13b7cd2f59 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/info/eaaa06c615f44200bce4fa13b7cd2f59 2024-11-09T02:21:30,919 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/.tmp/testFamily/6a268bbd8ca042a5a3f40b2a94471d2a is 61, key is row-0/testFamily:testQualifier/1731118721142/Put/seqid=0 2024-11-09T02:21:30,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741896_1072 (size=5049) 2024-11-09T02:21:30,927 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/info/eaaa06c615f44200bce4fa13b7cd2f59, entries=290, sequenceid=149, filesize=50.7 K 2024-11-09T02:21:30,928 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/.tmp/table/44e8388d183945a1b8bed4e53f90cb57 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/table/44e8388d183945a1b8bed4e53f90cb57 2024-11-09T02:21:30,935 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/table/44e8388d183945a1b8bed4e53f90cb57, entries=58, sequenceid=149, filesize=9.6 K 2024-11-09T02:21:30,936 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~47.95 KB/49100, heapSize ~74.06 KB/75840, currentSize=0 B/0 for 1588230740 in 572ms, sequenceid=149, compaction requested=false 2024-11-09T02:21:30,941 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/meta/1588230740/recovered.edits/152.seqid, newMaxSeqId=152, maxSeqId=1 2024-11-09T02:21:30,941 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:21:30,941 INFO [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T02:21:30,942 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T02:21:30,942 DEBUG [RS_CLOSE_META-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T02:21:30,965 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, 970ccc55dc0014f96f14429b0893f112, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:30,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:31,037 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync_1_-true' 2024-11-09T02:21:31,165 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, 970ccc55dc0014f96f14429b0893f112, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:31,326 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/.tmp/testFamily/6a268bbd8ca042a5a3f40b2a94471d2a 2024-11-09T02:21:31,333 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/.tmp/testFamily/6a268bbd8ca042a5a3f40b2a94471d2a as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/testFamily/6a268bbd8ca042a5a3f40b2a94471d2a 2024-11-09T02:21:31,339 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/testFamily/6a268bbd8ca042a5a3f40b2a94471d2a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:31,340 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 970ccc55dc0014f96f14429b0893f112 in 444ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:31,344 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_0_-false/970ccc55dc0014f96f14429b0893f112/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:31,344 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:21:31,344 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 970ccc55dc0014f96f14429b0893f112: 2024-11-09T02:21:31,344 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_0_-false,,1731118718116.970ccc55dc0014f96f14429b0893f112. 2024-11-09T02:21:31,345 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2452fe7575fea50ec376844daaf48eb5, disabling compactions & flushes 2024-11-09T02:21:31,345 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:21:31,345 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:21:31,345 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. after waiting 0 ms 2024-11-09T02:21:31,345 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:21:31,345 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2452fe7575fea50ec376844daaf48eb5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:31,360 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/.tmp/testFamily/6deb39a10b484fd2993ec5f295558863 is 61, key is row-0/testFamily:testQualifier/1731118793354/Put/seqid=0 2024-11-09T02:21:31,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741897_1073 (size=5049) 2024-11-09T02:21:31,366 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 27 regions to close 2024-11-09T02:21:31,366 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:31,566 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:31,765 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/.tmp/testFamily/6deb39a10b484fd2993ec5f295558863 2024-11-09T02:21:31,766 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2452fe7575fea50ec376844daaf48eb5, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:31,773 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/.tmp/testFamily/6deb39a10b484fd2993ec5f295558863 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/testFamily/6deb39a10b484fd2993ec5f295558863 2024-11-09T02:21:31,779 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/testFamily/6deb39a10b484fd2993ec5f295558863, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:31,779 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2452fe7575fea50ec376844daaf48eb5 in 434ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:31,783 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextExceptionAsync_1_-true/2452fe7575fea50ec376844daaf48eb5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:31,784 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2452fe7575fea50ec376844daaf48eb5: 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync_1_-true,,1731118790337.2452fe7575fea50ec376844daaf48eb5. 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2ac227a2fa5b5a4d941fcbac940cb5fb, disabling compactions & flushes 2024-11-09T02:21:31,784 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. after waiting 0 ms 2024-11-09T02:21:31,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:21:31,784 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2ac227a2fa5b5a4d941fcbac940cb5fb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:31,800 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/.tmp/testFamily/725a9f5305884b588ba54bb7008342d7 is 61, key is row-0/testFamily:testQualifier/1731118758033/Put/seqid=0 2024-11-09T02:21:31,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741898_1074 (size=5049) 2024-11-09T02:21:31,966 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:32,167 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2ac227a2fa5b5a4d941fcbac940cb5fb, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:32,205 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/.tmp/testFamily/725a9f5305884b588ba54bb7008342d7 2024-11-09T02:21:32,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/.tmp/testFamily/725a9f5305884b588ba54bb7008342d7 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/testFamily/725a9f5305884b588ba54bb7008342d7 2024-11-09T02:21:32,219 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/testFamily/725a9f5305884b588ba54bb7008342d7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:32,220 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ac227a2fa5b5a4d941fcbac940cb5fb in 436ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:32,223 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_0_-false/2ac227a2fa5b5a4d941fcbac940cb5fb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:32,224 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2ac227a2fa5b5a4d941fcbac940cb5fb: 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_0_-false,,1731118755015.2ac227a2fa5b5a4d941fcbac940cb5fb. 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 92e0494411be1d7676e60e07472411c6, disabling compactions & flushes 2024-11-09T02:21:32,224 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. after waiting 0 ms 2024-11-09T02:21:32,224 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:21:32,224 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 92e0494411be1d7676e60e07472411c6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:32,239 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/.tmp/testFamily/013b0f1fc70a4a25b1cb566319f96df8 is 61, key is row-0/testFamily:testQualifier/1731118753943/Put/seqid=0 2024-11-09T02:21:32,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741899_1075 (size=5049) 2024-11-09T02:21:32,367 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 25 regions to close 2024-11-09T02:21:32,367 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:32,399 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.RSRpcServices$ScannerListener(571): Scanner lease -1481684882995281785 expired clientIPAndPort=172.17.0.2:34738, userName=jenkins, regionInfo=testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:32,567 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, 92e0494411be1d7676e60e07472411c6, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:32,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:32,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true Metrics about Tables on a single HBase RegionServer 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_1_-true 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync_0_-false 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_0_-false 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_0_-false 2024-11-09T02:21:32,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:21:32,644 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/.tmp/testFamily/013b0f1fc70a4a25b1cb566319f96df8 2024-11-09T02:21:32,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/.tmp/testFamily/013b0f1fc70a4a25b1cb566319f96df8 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/testFamily/013b0f1fc70a4a25b1cb566319f96df8 2024-11-09T02:21:32,657 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/testFamily/013b0f1fc70a4a25b1cb566319f96df8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:32,658 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 92e0494411be1d7676e60e07472411c6 in 434ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:32,662 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_0_-false/92e0494411be1d7676e60e07472411c6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:32,663 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 92e0494411be1d7676e60e07472411c6: 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_0_-false,,1731118752921.92e0494411be1d7676e60e07472411c6. 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 27cb43c1981e47632a87e1850b149c1e, disabling compactions & flushes 2024-11-09T02:21:32,663 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. after waiting 0 ms 2024-11-09T02:21:32,663 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:21:32,663 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 27cb43c1981e47632a87e1850b149c1e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:32,679 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/.tmp/testFamily/da276f00c9b54bc1ae2833bf89a6eab6 is 61, key is row-0/testFamily:testQualifier/1731118709041/Put/seqid=0 2024-11-09T02:21:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741900_1076 (size=5049) 2024-11-09T02:21:32,767 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:32,968 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 27cb43c1981e47632a87e1850b149c1e, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,084 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/.tmp/testFamily/da276f00c9b54bc1ae2833bf89a6eab6 2024-11-09T02:21:33,092 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/.tmp/testFamily/da276f00c9b54bc1ae2833bf89a6eab6 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/testFamily/da276f00c9b54bc1ae2833bf89a6eab6 2024-11-09T02:21:33,097 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/testFamily/da276f00c9b54bc1ae2833bf89a6eab6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:33,098 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 27cb43c1981e47632a87e1850b149c1e in 435ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:33,102 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_0_-false/27cb43c1981e47632a87e1850b149c1e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:33,103 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 27cb43c1981e47632a87e1850b149c1e: 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_0_-false,,1731118708004.27cb43c1981e47632a87e1850b149c1e. 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing fd148266e0f7adabaf27b8fe8e091c94, disabling compactions & flushes 2024-11-09T02:21:33,103 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. after waiting 0 ms 2024-11-09T02:21:33,103 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:21:33,103 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing fd148266e0f7adabaf27b8fe8e091c94 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:33,119 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/.tmp/testFamily/b542c7763ac54477824a2893787ea1f1 is 61, key is row-0/testFamily:testQualifier/1731118751362/Put/seqid=0 2024-11-09T02:21:33,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741901_1077 (size=5049) 2024-11-09T02:21:33,168 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,368 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 23 regions to close 2024-11-09T02:21:33,368 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, fd148266e0f7adabaf27b8fe8e091c94, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/.tmp/testFamily/b542c7763ac54477824a2893787ea1f1 2024-11-09T02:21:33,532 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/.tmp/testFamily/b542c7763ac54477824a2893787ea1f1 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/testFamily/b542c7763ac54477824a2893787ea1f1 2024-11-09T02:21:33,538 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/testFamily/b542c7763ac54477824a2893787ea1f1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:33,539 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fd148266e0f7adabaf27b8fe8e091c94 in 436ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:33,543 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_0_-false/fd148266e0f7adabaf27b8fe8e091c94/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:33,543 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:21:33,543 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for fd148266e0f7adabaf27b8fe8e091c94: 2024-11-09T02:21:33,543 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_0_-false,,1731118750333.fd148266e0f7adabaf27b8fe8e091c94. 2024-11-09T02:21:33,543 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c899c96d16d3f8a690a93440304a00e0, disabling compactions & flushes 2024-11-09T02:21:33,544 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:33,544 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:33,544 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. after waiting 0 ms 2024-11-09T02:21:33,544 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:33,544 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c899c96d16d3f8a690a93440304a00e0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:33,561 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/.tmp/testFamily/ff28088768ec436fb7d2826f2ea2a9b4 is 61, key is row-0/testFamily:testQualifier/1731118863204/Put/seqid=0 2024-11-09T02:21:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741902_1078 (size=5049) 2024-11-09T02:21:33,568 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,769 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,965 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/.tmp/testFamily/ff28088768ec436fb7d2826f2ea2a9b4 2024-11-09T02:21:33,969 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, c899c96d16d3f8a690a93440304a00e0, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:33,973 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/.tmp/testFamily/ff28088768ec436fb7d2826f2ea2a9b4 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/testFamily/ff28088768ec436fb7d2826f2ea2a9b4 2024-11-09T02:21:33,979 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/testFamily/ff28088768ec436fb7d2826f2ea2a9b4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:33,983 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c899c96d16d3f8a690a93440304a00e0 in 439ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:33,988 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_1_-true/c899c96d16d3f8a690a93440304a00e0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:33,988 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:33,988 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c899c96d16d3f8a690a93440304a00e0: 2024-11-09T02:21:33,989 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_1_-true,,1731118860189.c899c96d16d3f8a690a93440304a00e0. 2024-11-09T02:21:33,989 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ff049946e3d00cdf907ccf7cd7ae4297, disabling compactions & flushes 2024-11-09T02:21:33,989 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:21:33,989 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:21:33,989 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. after waiting 0 ms 2024-11-09T02:21:33,989 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:21:33,989 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ff049946e3d00cdf907ccf7cd7ae4297 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:34,007 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/.tmp/testFamily/f1543ddf55764786858e215260f39522 is 61, key is row-0/testFamily:testQualifier/1731118787295/Put/seqid=0 2024-11-09T02:21:34,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741903_1079 (size=5049) 2024-11-09T02:21:34,169 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:34,369 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 21 regions to close 2024-11-09T02:21:34,370 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183, ff049946e3d00cdf907ccf7cd7ae4297 2024-11-09T02:21:34,412 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/.tmp/testFamily/f1543ddf55764786858e215260f39522 2024-11-09T02:21:34,420 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/.tmp/testFamily/f1543ddf55764786858e215260f39522 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/testFamily/f1543ddf55764786858e215260f39522 2024-11-09T02:21:34,427 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/testFamily/f1543ddf55764786858e215260f39522, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:34,428 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ff049946e3d00cdf907ccf7cd7ae4297 in 439ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:34,432 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_0_-false/ff049946e3d00cdf907ccf7cd7ae4297/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:34,432 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:21:34,432 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ff049946e3d00cdf907ccf7cd7ae4297: 2024-11-09T02:21:34,433 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_0_-false,,1731118784278.ff049946e3d00cdf907ccf7cd7ae4297. 2024-11-09T02:21:34,433 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 755bcccae28504c223e008c40b24c6c4, disabling compactions & flushes 2024-11-09T02:21:34,433 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:21:34,433 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:21:34,433 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. after waiting 0 ms 2024-11-09T02:21:34,433 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:21:34,433 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 755bcccae28504c223e008c40b24c6c4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:34,448 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/.tmp/testFamily/676e3ef3b1ed47fc81b7a798596b9549 is 61, key is row-0/testFamily:testQualifier/1731118778217/Put/seqid=0 2024-11-09T02:21:34,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741904_1080 (size=5049) 2024-11-09T02:21:34,570 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:34,770 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 755bcccae28504c223e008c40b24c6c4, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:34,854 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/.tmp/testFamily/676e3ef3b1ed47fc81b7a798596b9549 2024-11-09T02:21:34,861 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/.tmp/testFamily/676e3ef3b1ed47fc81b7a798596b9549 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/testFamily/676e3ef3b1ed47fc81b7a798596b9549 2024-11-09T02:21:34,867 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/testFamily/676e3ef3b1ed47fc81b7a798596b9549, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:34,868 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 755bcccae28504c223e008c40b24c6c4 in 435ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:34,872 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_0_-false/755bcccae28504c223e008c40b24c6c4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:34,873 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 755bcccae28504c223e008c40b24c6c4: 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_0_-false,,1731118775191.755bcccae28504c223e008c40b24c6c4. 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 37354a7c4ac00e13f36e6363dec270fb, disabling compactions & flushes 2024-11-09T02:21:34,873 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. after waiting 0 ms 2024-11-09T02:21:34,873 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:21:34,873 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 37354a7c4ac00e13f36e6363dec270fb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:34,896 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/.tmp/testFamily/6bd3a36c581d400483b197d4a60f1824 is 61, key is row-0/testFamily:testQualifier/1731118853134/Put/seqid=0 2024-11-09T02:21:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741905_1081 (size=5049) 2024-11-09T02:21:34,970 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:35,171 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 37354a7c4ac00e13f36e6363dec270fb, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:35,301 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/.tmp/testFamily/6bd3a36c581d400483b197d4a60f1824 2024-11-09T02:21:35,308 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/.tmp/testFamily/6bd3a36c581d400483b197d4a60f1824 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/testFamily/6bd3a36c581d400483b197d4a60f1824 2024-11-09T02:21:35,314 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/testFamily/6bd3a36c581d400483b197d4a60f1824, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:35,315 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 37354a7c4ac00e13f36e6363dec270fb in 442ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:35,319 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnRenewLease_1_-true/37354a7c4ac00e13f36e6363dec270fb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:35,319 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:21:35,319 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 37354a7c4ac00e13f36e6363dec270fb: 2024-11-09T02:21:35,319 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnRenewLease_1_-true,,1731118850115.37354a7c4ac00e13f36e6363dec270fb. 2024-11-09T02:21:35,320 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 6235bfc49272e5114ec5c10328305fa2, disabling compactions & flushes 2024-11-09T02:21:35,320 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:21:35,320 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:21:35,320 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. after waiting 0 ms 2024-11-09T02:21:35,320 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:21:35,320 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 6235bfc49272e5114ec5c10328305fa2 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-09T02:21:35,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/.tmp/info/e500b6c6d19546f2903c02c7d908b200 is 45, key is default/info:d/1731118706431/Put/seqid=0 2024-11-09T02:21:35,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741906_1082 (size=5037) 2024-11-09T02:21:35,371 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 18 regions to close 2024-11-09T02:21:35,371 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:35,523 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(140): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=54f3333e9615:40839 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.getLiveRegionServers(RegionServerStatusProtos.java:17362) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionImplementation.getLiveRegionServers(ConnectionImplementation.java:2376) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:138) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 12 more 2024-11-09T02:21:35,571 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 6235bfc49272e5114ec5c10328305fa2, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:35,741 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/.tmp/info/e500b6c6d19546f2903c02c7d908b200 2024-11-09T02:21:35,748 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/.tmp/info/e500b6c6d19546f2903c02c7d908b200 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/info/e500b6c6d19546f2903c02c7d908b200 2024-11-09T02:21:35,755 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/info/e500b6c6d19546f2903c02c7d908b200, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T02:21:35,756 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 6235bfc49272e5114ec5c10328305fa2 in 436ms, sequenceid=6, compaction requested=false 2024-11-09T02:21:35,761 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/namespace/6235bfc49272e5114ec5c10328305fa2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T02:21:35,761 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:21:35,761 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 6235bfc49272e5114ec5c10328305fa2: 2024-11-09T02:21:35,761 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731118705366.6235bfc49272e5114ec5c10328305fa2. 2024-11-09T02:21:35,762 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f0e7baf77f7cbbc7df2c5cf980de6183, disabling compactions & flushes 2024-11-09T02:21:35,762 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:35,762 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:35,762 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. after waiting 0 ms 2024-11-09T02:21:35,762 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:35,762 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing f0e7baf77f7cbbc7df2c5cf980de6183 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:35,772 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:35,784 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/.tmp/testFamily/d770cf63b709417eaea58ec80dcee8a9 is 61, key is row-0/testFamily:testQualifier/1731118828850/Put/seqid=0 2024-11-09T02:21:35,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741907_1083 (size=5049) 2024-11-09T02:21:35,972 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:36,172 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0, f0e7baf77f7cbbc7df2c5cf980de6183 2024-11-09T02:21:36,189 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/.tmp/testFamily/d770cf63b709417eaea58ec80dcee8a9 2024-11-09T02:21:36,196 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/.tmp/testFamily/d770cf63b709417eaea58ec80dcee8a9 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/testFamily/d770cf63b709417eaea58ec80dcee8a9 2024-11-09T02:21:36,201 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/testFamily/d770cf63b709417eaea58ec80dcee8a9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:36,202 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f0e7baf77f7cbbc7df2c5cf980de6183 in 440ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:36,206 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_1_-true/f0e7baf77f7cbbc7df2c5cf980de6183/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:36,206 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:36,206 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f0e7baf77f7cbbc7df2c5cf980de6183: 2024-11-09T02:21:36,206 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_1_-true,,1731118825835.f0e7baf77f7cbbc7df2c5cf980de6183. 2024-11-09T02:21:36,207 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 222cbbc69904e90550664cf8da7b9c60, disabling compactions & flushes 2024-11-09T02:21:36,207 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:36,207 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:36,207 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. after waiting 0 ms 2024-11-09T02:21:36,207 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:36,207 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 222cbbc69904e90550664cf8da7b9c60 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:36,222 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/.tmp/testFamily/cbaa7d4339c74bc79db5aaa199cd99da is 61, key is row-0/testFamily:testQualifier/1731118887330/Put/seqid=0 2024-11-09T02:21:36,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741908_1084 (size=5049) 2024-11-09T02:21:36,372 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 16 regions to close 2024-11-09T02:21:36,373 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:36,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:36,573 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 222cbbc69904e90550664cf8da7b9c60, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:36,627 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/.tmp/testFamily/cbaa7d4339c74bc79db5aaa199cd99da 2024-11-09T02:21:36,634 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/.tmp/testFamily/cbaa7d4339c74bc79db5aaa199cd99da as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/testFamily/cbaa7d4339c74bc79db5aaa199cd99da 2024-11-09T02:21:36,640 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/testFamily/cbaa7d4339c74bc79db5aaa199cd99da, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:36,640 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 222cbbc69904e90550664cf8da7b9c60 in 433ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:36,644 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScannerAsync_1_-true/222cbbc69904e90550664cf8da7b9c60/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:36,645 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 222cbbc69904e90550664cf8da7b9c60: 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync_1_-true,,1731118884312.222cbbc69904e90550664cf8da7b9c60. 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 65c9e222499018697aa1704e3adfc51e, disabling compactions & flushes 2024-11-09T02:21:36,645 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. after waiting 0 ms 2024-11-09T02:21:36,645 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:36,645 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 65c9e222499018697aa1704e3adfc51e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:36,661 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/.tmp/testFamily/d2d79960201848b28fd047079fdfae9c is 61, key is row-0/testFamily:testQualifier/1731118875266/Put/seqid=0 2024-11-09T02:21:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741909_1085 (size=5049) 2024-11-09T02:21:36,773 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:36,973 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 65c9e222499018697aa1704e3adfc51e, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,066 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/.tmp/testFamily/d2d79960201848b28fd047079fdfae9c 2024-11-09T02:21:37,073 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/.tmp/testFamily/d2d79960201848b28fd047079fdfae9c as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/testFamily/d2d79960201848b28fd047079fdfae9c 2024-11-09T02:21:37,079 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/testFamily/d2d79960201848b28fd047079fdfae9c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:37,080 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65c9e222499018697aa1704e3adfc51e in 435ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:37,084 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNext_1_-true/65c9e222499018697aa1704e3adfc51e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:37,085 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 65c9e222499018697aa1704e3adfc51e: 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext_1_-true,,1731118872251.65c9e222499018697aa1704e3adfc51e. 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 1e9b57e787ebaa31f7960f0e0be0fb6f, disabling compactions & flushes 2024-11-09T02:21:37,085 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. after waiting 0 ms 2024-11-09T02:21:37,085 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:21:37,085 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 1e9b57e787ebaa31f7960f0e0be0fb6f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:37,100 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/.tmp/testFamily/486426a5514946848e5ccc339d52194f is 61, key is row-0/testFamily:testQualifier/1731118739557/Put/seqid=0 2024-11-09T02:21:37,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741910_1086 (size=5049) 2024-11-09T02:21:37,174 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,374 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 14 regions to close 2024-11-09T02:21:37,374 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 1e9b57e787ebaa31f7960f0e0be0fb6f, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,506 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/.tmp/testFamily/486426a5514946848e5ccc339d52194f 2024-11-09T02:21:37,513 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/.tmp/testFamily/486426a5514946848e5ccc339d52194f as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/testFamily/486426a5514946848e5ccc339d52194f 2024-11-09T02:21:37,518 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/testFamily/486426a5514946848e5ccc339d52194f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:37,519 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1e9b57e787ebaa31f7960f0e0be0fb6f in 434ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:37,523 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNextAsync_0_-false/1e9b57e787ebaa31f7960f0e0be0fb6f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:37,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 1e9b57e787ebaa31f7960f0e0be0fb6f: 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync_0_-false,,1731118738532.1e9b57e787ebaa31f7960f0e0be0fb6f. 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3152c4d9a734cd45a64adc4eaba61986, disabling compactions & flushes 2024-11-09T02:21:37,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. after waiting 0 ms 2024-11-09T02:21:37,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:21:37,525 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3152c4d9a734cd45a64adc4eaba61986 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:37,543 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/.tmp/testFamily/0949954ef36d46e1bf99224d78515cc7 is 61, key is row-0/testFamily:testQualifier/1731118766133/Put/seqid=0 2024-11-09T02:21:37,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741911_1087 (size=5049) 2024-11-09T02:21:37,574 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,774 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 3152c4d9a734cd45a64adc4eaba61986, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,949 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/.tmp/testFamily/0949954ef36d46e1bf99224d78515cc7 2024-11-09T02:21:37,956 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/.tmp/testFamily/0949954ef36d46e1bf99224d78515cc7 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/testFamily/0949954ef36d46e1bf99224d78515cc7 2024-11-09T02:21:37,962 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/testFamily/0949954ef36d46e1bf99224d78515cc7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:37,963 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3152c4d9a734cd45a64adc4eaba61986 in 438ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:37,967 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScannerAsync_0_-false/3152c4d9a734cd45a64adc4eaba61986/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:37,967 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:21:37,967 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3152c4d9a734cd45a64adc4eaba61986: 2024-11-09T02:21:37,968 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync_0_-false,,1731118765109.3152c4d9a734cd45a64adc4eaba61986. 2024-11-09T02:21:37,968 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 04a6b35baf34def60d1b7d01a8823d80, disabling compactions & flushes 2024-11-09T02:21:37,968 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:21:37,968 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:21:37,968 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. after waiting 0 ms 2024-11-09T02:21:37,968 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:21:37,968 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 04a6b35baf34def60d1b7d01a8823d80 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:37,975 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:37,983 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/.tmp/testFamily/704e5d327721437c87b3dcf39b0f913f is 61, key is row-0/testFamily:testQualifier/1731118732311/Put/seqid=0 2024-11-09T02:21:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741912_1088 (size=5049) 2024-11-09T02:21:38,175 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:38,375 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 12 regions to close 2024-11-09T02:21:38,375 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 04a6b35baf34def60d1b7d01a8823d80, 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:38,388 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/.tmp/testFamily/704e5d327721437c87b3dcf39b0f913f 2024-11-09T02:21:38,395 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/.tmp/testFamily/704e5d327721437c87b3dcf39b0f913f as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/testFamily/704e5d327721437c87b3dcf39b0f913f 2024-11-09T02:21:38,402 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/testFamily/704e5d327721437c87b3dcf39b0f913f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:38,403 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 04a6b35baf34def60d1b7d01a8823d80 in 435ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:38,407 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_0_-false/04a6b35baf34def60d1b7d01a8823d80/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:38,408 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 04a6b35baf34def60d1b7d01a8823d80: 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_0_-false,,1731118731289.04a6b35baf34def60d1b7d01a8823d80. 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ce03aa7d3b74ad9aaf2d966daa46e587, disabling compactions & flushes 2024-11-09T02:21:38,408 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. after waiting 0 ms 2024-11-09T02:21:38,408 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:21:38,408 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ce03aa7d3b74ad9aaf2d966daa46e587 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:38,431 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/.tmp/testFamily/70c8f93807ea4e48b3eac5c7a9f73e7c is 61, key is row-0/testFamily:testQualifier/1731118735457/Put/seqid=0 2024-11-09T02:21:38,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741913_1089 (size=5049) 2024-11-09T02:21:38,575 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:38,776 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, ce03aa7d3b74ad9aaf2d966daa46e587, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:38,837 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/.tmp/testFamily/70c8f93807ea4e48b3eac5c7a9f73e7c 2024-11-09T02:21:38,844 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/.tmp/testFamily/70c8f93807ea4e48b3eac5c7a9f73e7c as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/testFamily/70c8f93807ea4e48b3eac5c7a9f73e7c 2024-11-09T02:21:38,850 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/testFamily/70c8f93807ea4e48b3eac5c7a9f73e7c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:38,851 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ce03aa7d3b74ad9aaf2d966daa46e587 in 443ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:38,855 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_0_-false/ce03aa7d3b74ad9aaf2d966daa46e587/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:38,856 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ce03aa7d3b74ad9aaf2d966daa46e587: 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_0_-false,,1731118734406.ce03aa7d3b74ad9aaf2d966daa46e587. 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 69b0fe0f891da7370283b06dbbed5543, disabling compactions & flushes 2024-11-09T02:21:38,856 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. after waiting 0 ms 2024-11-09T02:21:38,856 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:38,856 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 69b0fe0f891da7370283b06dbbed5543 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:38,879 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/.tmp/testFamily/575f6ccd0ee34480b8e039d74c154162 is 61, key is row-0/testFamily:testQualifier/1731118806505/Put/seqid=0 2024-11-09T02:21:38,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741914_1090 (size=5049) 2024-11-09T02:21:38,976 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:39,176 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 69b0fe0f891da7370283b06dbbed5543, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, ac58767bfb05dc96875b60142d320308, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:39,285 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/.tmp/testFamily/575f6ccd0ee34480b8e039d74c154162 2024-11-09T02:21:39,291 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/.tmp/testFamily/575f6ccd0ee34480b8e039d74c154162 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/testFamily/575f6ccd0ee34480b8e039d74c154162 2024-11-09T02:21:39,296 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/testFamily/575f6ccd0ee34480b8e039d74c154162, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:39,297 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69b0fe0f891da7370283b06dbbed5543 in 441ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnNextAsync_1_-true/69b0fe0f891da7370283b06dbbed5543/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:39,301 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 69b0fe0f891da7370283b06dbbed5543: 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync_1_-true,,1731118805486.69b0fe0f891da7370283b06dbbed5543. 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing ac58767bfb05dc96875b60142d320308, disabling compactions & flushes 2024-11-09T02:21:39,301 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. after waiting 0 ms 2024-11-09T02:21:39,301 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:21:39,301 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing ac58767bfb05dc96875b60142d320308 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:39,316 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/.tmp/testFamily/e85ad2e68b43402bbbed786ccce8a1eb is 61, key is row-0/testFamily:testQualifier/1731118731227/Put/seqid=0 2024-11-09T02:21:39,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741915_1091 (size=5049) 2024-11-09T02:21:39,320 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/.tmp/testFamily/e85ad2e68b43402bbbed786ccce8a1eb 2024-11-09T02:21:39,326 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/.tmp/testFamily/e85ad2e68b43402bbbed786ccce8a1eb as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/testFamily/e85ad2e68b43402bbbed786ccce8a1eb 2024-11-09T02:21:39,331 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/testFamily/e85ad2e68b43402bbbed786ccce8a1eb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:39,332 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ac58767bfb05dc96875b60142d320308 in 31ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:39,335 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_0_-false/ac58767bfb05dc96875b60142d320308/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:39,336 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for ac58767bfb05dc96875b60142d320308: 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_0_-false,,1731118730207.ac58767bfb05dc96875b60142d320308. 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 15f40df06f118f644828b6a5fdc8e9e9, disabling compactions & flushes 2024-11-09T02:21:39,336 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. after waiting 0 ms 2024-11-09T02:21:39,336 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:39,336 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 15f40df06f118f644828b6a5fdc8e9e9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:39,351 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/.tmp/testFamily/082727dd01194bdb9403306d02dff790 is 61, key is row-0/testFamily:testQualifier/1731118822789/Put/seqid=0 2024-11-09T02:21:39,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741916_1092 (size=5049) 2024-11-09T02:21:39,376 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 8 regions to close 2024-11-09T02:21:39,376 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1603): Online Regions={15f40df06f118f644828b6a5fdc8e9e9=testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9., dafbab47240d2184eb5b053eb17e581a=testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a., 647c800ac4cc2c15506f689be4cf8b8f=testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f., 8efa319d73410a0150970c2107175f31=hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31., 2b58f24cf912d4e547dcedf3080490dd=hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd., e96580184280b371fd9cd3867da5c0c0=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., d5566be66cd3b6f502d985d4b9c532a5=testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5., 8e36f6e4daeb83f83d15ea0b8a72297d=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.} 2024-11-09T02:21:39,377 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:39,577 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 15f40df06f118f644828b6a5fdc8e9e9, 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:39,755 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/.tmp/testFamily/082727dd01194bdb9403306d02dff790 2024-11-09T02:21:39,763 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/.tmp/testFamily/082727dd01194bdb9403306d02dff790 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/testFamily/082727dd01194bdb9403306d02dff790 2024-11-09T02:21:39,768 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/testFamily/082727dd01194bdb9403306d02dff790, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:39,769 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 15f40df06f118f644828b6a5fdc8e9e9 in 433ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:39,773 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnOpenScanner_1_-true/15f40df06f118f644828b6a5fdc8e9e9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:39,773 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 15f40df06f118f644828b6a5fdc8e9e9: 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner_1_-true,,1731118819774.15f40df06f118f644828b6a5fdc8e9e9. 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing dafbab47240d2184eb5b053eb17e581a, disabling compactions & flushes 2024-11-09T02:21:39,774 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. after waiting 0 ms 2024-11-09T02:21:39,774 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:21:39,774 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing dafbab47240d2184eb5b053eb17e581a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:39,777 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:39,791 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/.tmp/testFamily/f7df58b5e68c4a6cb6da25638ca5711d is 61, key is row-0/testFamily:testQualifier/1731118842514/Put/seqid=0 2024-11-09T02:21:39,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741917_1093 (size=5049) 2024-11-09T02:21:39,977 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:40,178 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, dafbab47240d2184eb5b053eb17e581a, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:40,196 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/.tmp/testFamily/f7df58b5e68c4a6cb6da25638ca5711d 2024-11-09T02:21:40,202 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/.tmp/testFamily/f7df58b5e68c4a6cb6da25638ca5711d as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/testFamily/f7df58b5e68c4a6cb6da25638ca5711d 2024-11-09T02:21:40,207 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/testFamily/f7df58b5e68c4a6cb6da25638ca5711d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:40,208 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dafbab47240d2184eb5b053eb17e581a in 434ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:40,212 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnClose_1_-true/dafbab47240d2184eb5b053eb17e581a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:40,213 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for dafbab47240d2184eb5b053eb17e581a: 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnClose_1_-true,,1731118839498.dafbab47240d2184eb5b053eb17e581a. 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 647c800ac4cc2c15506f689be4cf8b8f, disabling compactions & flushes 2024-11-09T02:21:40,213 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. after waiting 0 ms 2024-11-09T02:21:40,213 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:40,213 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 647c800ac4cc2c15506f689be4cf8b8f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:40,228 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/.tmp/testFamily/183d46bb62e046059adf6d4d184471b8 is 61, key is row-0/testFamily:testQualifier/1731118816600/Put/seqid=0 2024-11-09T02:21:40,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741918_1094 (size=5049) 2024-11-09T02:21:40,378 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 6 regions to close 2024-11-09T02:21:40,378 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1603): Online Regions={647c800ac4cc2c15506f689be4cf8b8f=testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f., 8efa319d73410a0150970c2107175f31=hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31., 2b58f24cf912d4e547dcedf3080490dd=hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd., e96580184280b371fd9cd3867da5c0c0=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., d5566be66cd3b6f502d985d4b9c532a5=testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5., 8e36f6e4daeb83f83d15ea0b8a72297d=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.} 2024-11-09T02:21:40,378 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:40,578 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 647c800ac4cc2c15506f689be4cf8b8f, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:40,633 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/.tmp/testFamily/183d46bb62e046059adf6d4d184471b8 2024-11-09T02:21:40,639 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/.tmp/testFamily/183d46bb62e046059adf6d4d184471b8 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/testFamily/183d46bb62e046059adf6d4d184471b8 2024-11-09T02:21:40,645 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/testFamily/183d46bb62e046059adf6d4d184471b8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:40,646 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 647c800ac4cc2c15506f689be4cf8b8f in 433ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:40,650 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testRetryOutOfOrderScannerNextException_1_-true/647c800ac4cc2c15506f689be4cf8b8f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:40,650 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:40,650 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 647c800ac4cc2c15506f689be4cf8b8f: 2024-11-09T02:21:40,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException_1_-true,,1731118815569.647c800ac4cc2c15506f689be4cf8b8f. 2024-11-09T02:21:40,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8efa319d73410a0150970c2107175f31, disabling compactions & flushes 2024-11-09T02:21:40,651 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:21:40,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:21:40,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. after waiting 0 ms 2024-11-09T02:21:40,651 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:21:40,651 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8efa319d73410a0150970c2107175f31 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:40,666 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/.tmp/testFamily/b5abeb2bec4743ad92e90814cbfa762e is 61, key is row-0/testFamily:testQualifier/1731118743752/Put/seqid=0 2024-11-09T02:21:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741919_1095 (size=5049) 2024-11-09T02:21:40,778 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:40,979 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 8e36f6e4daeb83f83d15ea0b8a72297d, 8efa319d73410a0150970c2107175f31, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:41,070 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/.tmp/testFamily/b5abeb2bec4743ad92e90814cbfa762e 2024-11-09T02:21:41,077 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/.tmp/testFamily/b5abeb2bec4743ad92e90814cbfa762e as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/testFamily/b5abeb2bec4743ad92e90814cbfa762e 2024-11-09T02:21:41,082 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/testFamily/b5abeb2bec4743ad92e90814cbfa762e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:41,083 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8efa319d73410a0150970c2107175f31 in 431ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:41,086 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_0_-false/8efa319d73410a0150970c2107175f31/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:41,086 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:21:41,086 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8efa319d73410a0150970c2107175f31: 2024-11-09T02:21:41,086 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_0_-false,,1731118742681.8efa319d73410a0150970c2107175f31. 2024-11-09T02:21:41,087 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 2b58f24cf912d4e547dcedf3080490dd, disabling compactions & flushes 2024-11-09T02:21:41,087 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:21:41,087 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:21:41,087 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. after waiting 0 ms 2024-11-09T02:21:41,087 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:21:41,087 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 2b58f24cf912d4e547dcedf3080490dd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:41,102 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/.tmp/testFamily/b6a23ee897da4683a638b3e7c425b824 is 61, key is row-0/testFamily:testQualifier/1731118832941/Put/seqid=0 2024-11-09T02:21:41,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741920_1096 (size=5049) 2024-11-09T02:21:41,179 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:41,379 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close 2024-11-09T02:21:41,379 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1603): Online Regions={2b58f24cf912d4e547dcedf3080490dd=hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd., e96580184280b371fd9cd3867da5c0c0=hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0., d5566be66cd3b6f502d985d4b9c532a5=testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5., 8e36f6e4daeb83f83d15ea0b8a72297d=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.} 2024-11-09T02:21:41,379 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 2b58f24cf912d4e547dcedf3080490dd, 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:41,507 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/.tmp/testFamily/b6a23ee897da4683a638b3e7c425b824 2024-11-09T02:21:41,513 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/.tmp/testFamily/b6a23ee897da4683a638b3e7c425b824 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/testFamily/b6a23ee897da4683a638b3e7c425b824 2024-11-09T02:21:41,519 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/testFamily/b6a23ee897da4683a638b3e7c425b824, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:41,520 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2b58f24cf912d4e547dcedf3080490dd in 433ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:41,523 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnClose_1_-true/2b58f24cf912d4e547dcedf3080490dd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:41,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 2b58f24cf912d4e547dcedf3080490dd: 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnClose_1_-true,,1731118831900.2b58f24cf912d4e547dcedf3080490dd. 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing e96580184280b371fd9cd3867da5c0c0, disabling compactions & flushes 2024-11-09T02:21:41,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. after waiting 0 ms 2024-11-09T02:21:41,524 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:41,524 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing e96580184280b371fd9cd3867da5c0c0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:41,539 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/.tmp/testFamily/cab5f8898bbb486691f6297bf6195c40 is 61, key is row-0/testFamily:testQualifier/1731118796426/Put/seqid=0 2024-11-09T02:21:41,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741921_1097 (size=5049) 2024-11-09T02:21:41,580 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:41,708 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:21:41,780 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5, e96580184280b371fd9cd3867da5c0c0 2024-11-09T02:21:41,945 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/.tmp/testFamily/cab5f8898bbb486691f6297bf6195c40 2024-11-09T02:21:41,951 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/.tmp/testFamily/cab5f8898bbb486691f6297bf6195c40 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/testFamily/cab5f8898bbb486691f6297bf6195c40 2024-11-09T02:21:41,957 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/testFamily/cab5f8898bbb486691f6297bf6195c40, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:41,958 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e96580184280b371fd9cd3867da5c0c0 in 434ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:41,962 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/hbase/testMetaScanTimeoutOnOpenScanner_1_-true/e96580184280b371fd9cd3867da5c0c0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:41,963 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for e96580184280b371fd9cd3867da5c0c0: 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner_1_-true,,1731118793408.e96580184280b371fd9cd3867da5c0c0. 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d5566be66cd3b6f502d985d4b9c532a5, disabling compactions & flushes 2024-11-09T02:21:41,963 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. after waiting 0 ms 2024-11-09T02:21:41,963 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:41,964 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing d5566be66cd3b6f502d985d4b9c532a5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:41,979 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/.tmp/testFamily/0deaf78a893d4ff5b0a97268b1288849 is 61, key is row-0/testFamily:testQualifier/1731118817712/Put/seqid=0 2024-11-09T02:21:41,980 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:21:41,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741922_1098 (size=5049) 2024-11-09T02:21:42,180 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:21:42,262 ERROR [LeaseRenewer:jenkins@localhost:40811 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:40811,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:21:42,380 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-09T02:21:42,380 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1603): Online Regions={d5566be66cd3b6f502d985d4b9c532a5=testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5., 8e36f6e4daeb83f83d15ea0b8a72297d=testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d.} 2024-11-09T02:21:42,380 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d, d5566be66cd3b6f502d985d4b9c532a5 2024-11-09T02:21:42,384 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/.tmp/testFamily/0deaf78a893d4ff5b0a97268b1288849 2024-11-09T02:21:42,391 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/.tmp/testFamily/0deaf78a893d4ff5b0a97268b1288849 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/testFamily/0deaf78a893d4ff5b0a97268b1288849 2024-11-09T02:21:42,396 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/testFamily/0deaf78a893d4ff5b0a97268b1288849, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:42,397 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d5566be66cd3b6f502d985d4b9c532a5 in 433ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:42,401 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnRenewLease_1_-true/d5566be66cd3b6f502d985d4b9c532a5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:42,401 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:42,401 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d5566be66cd3b6f502d985d4b9c532a5: 2024-11-09T02:21:42,401 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnRenewLease_1_-true,,1731118816691.d5566be66cd3b6f502d985d4b9c532a5. 2024-11-09T02:21:42,402 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8e36f6e4daeb83f83d15ea0b8a72297d, disabling compactions & flushes 2024-11-09T02:21:42,402 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:21:42,402 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:21:42,402 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. after waiting 0 ms 2024-11-09T02:21:42,402 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:21:42,402 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8e36f6e4daeb83f83d15ea0b8a72297d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T02:21:42,417 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/.tmp/testFamily/d9dc1612971248a1b60eec360d609970 is 61, key is row-0/testFamily:testQualifier/1731118847070/Put/seqid=0 2024-11-09T02:21:42,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741923_1099 (size=5049) 2024-11-09T02:21:42,581 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnRenewLease_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnClose_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext_0_-false 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnRenewLease_1_-true 2024-11-09T02:21:42,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnClose_0_-false 2024-11-09T02:21:42,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync_1_-true 2024-11-09T02:21:42,781 DEBUG [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1629): Waiting on 8e36f6e4daeb83f83d15ea0b8a72297d 2024-11-09T02:21:42,822 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/.tmp/testFamily/d9dc1612971248a1b60eec360d609970 2024-11-09T02:21:42,828 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/.tmp/testFamily/d9dc1612971248a1b60eec360d609970 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/testFamily/d9dc1612971248a1b60eec360d609970 2024-11-09T02:21:42,833 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/testFamily/d9dc1612971248a1b60eec360d609970, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T02:21:42,834 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8e36f6e4daeb83f83d15ea0b8a72297d in 432ms, sequenceid=8, compaction requested=false 2024-11-09T02:21:42,838 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/data/default/testNormalScanTimeoutOnNext_1_-true/8e36f6e4daeb83f83d15ea0b8a72297d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T02:21:42,839 INFO [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:21:42,839 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8e36f6e4daeb83f83d15ea0b8a72297d: 2024-11-09T02:21:42,839 DEBUG [RS_CLOSE_REGION-regionserver/54f3333e9615:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext_1_-true,,1731118844053.8e36f6e4daeb83f83d15ea0b8a72297d. 2024-11-09T02:21:42,981 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1250): stopping server 54f3333e9615,45997,1731118672827; all regions closed. 2024-11-09T02:21:42,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741834_1010 (size=56731) 2024-11-09T02:21:42,987 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(725): complete file /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/WALs/54f3333e9615,45997,1731118672827/54f3333e9615%2C45997%2C1731118672827.meta.1731118705122.meta not finished, retry = 0 2024-11-09T02:21:43,092 DEBUG [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/oldWALs 2024-11-09T02:21:43,093 INFO [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 54f3333e9615%2C45997%2C1731118672827.meta:.meta(num 1731118705122) 2024-11-09T02:21:43,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741833_1009 (size=60562) 2024-11-09T02:21:43,097 DEBUG [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/oldWALs 2024-11-09T02:21:43,097 INFO [RS:0;54f3333e9615:45997 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 54f3333e9615%2C45997%2C1731118672827:(num 1731118704749) 2024-11-09T02:21:43,097 DEBUG [RS:0;54f3333e9615:45997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:43,097 INFO [RS:0;54f3333e9615:45997 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:21:43,098 INFO [RS:0;54f3333e9615:45997 {}] hbase.ChoreService(370): Chore service for: regionserver/54f3333e9615:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-11-09T02:21:43,098 INFO [regionserver/54f3333e9615:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T02:21:43,099 INFO [RS:0;54f3333e9615:45997 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45997 2024-11-09T02:21:43,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/54f3333e9615,45997,1731118672827 2024-11-09T02:21:43,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:21:43,103 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$359/0x00007f56b08d4010@60281d8a rejected from java.util.concurrent.ThreadPoolExecutor@252a8feb[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-09T02:21:43,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [54f3333e9615,45997,1731118672827] 2024-11-09T02:21:43,105 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 54f3333e9615,45997,1731118672827; numProcessing=1 2024-11-09T02:21:43,106 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/54f3333e9615,45997,1731118672827 already deleted, retry=false 2024-11-09T02:21:43,106 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 54f3333e9615,45997,1731118672827 expired; onlineServers=0 2024-11-09T02:21:43,106 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '54f3333e9615,40839,1731118672115' ***** 2024-11-09T02:21:43,106 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T02:21:43,106 DEBUG [M:0;54f3333e9615:40839 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25273ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54f3333e9615/172.17.0.2:0 2024-11-09T02:21:43,106 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegionServer(1224): stopping server 54f3333e9615,40839,1731118672115 2024-11-09T02:21:43,106 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegionServer(1250): stopping server 54f3333e9615,40839,1731118672115; all regions closed. 2024-11-09T02:21:43,106 DEBUG [M:0;54f3333e9615:40839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:21:43,107 DEBUG [M:0;54f3333e9615:40839 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T02:21:43,107 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T02:21:43,107 DEBUG [M:0;54f3333e9615:40839 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T02:21:43,107 DEBUG [master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.small.0-1731118674508 {}] cleaner.HFileCleaner(306): Exit Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.small.0-1731118674508,5,FailOnTimeoutGroup] 2024-11-09T02:21:43,107 DEBUG [master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.large.0-1731118674505 {}] cleaner.HFileCleaner(306): Exit Thread[master/54f3333e9615:0:becomeActiveMaster-HFileCleaner.large.0-1731118674505,5,FailOnTimeoutGroup] 2024-11-09T02:21:43,107 INFO [M:0;54f3333e9615:40839 {}] hbase.ChoreService(370): Chore service for: master/54f3333e9615:0 had [] on shutdown 2024-11-09T02:21:43,107 DEBUG [M:0;54f3333e9615:40839 {}] master.HMaster(1733): Stopping service threads 2024-11-09T02:21:43,107 INFO [M:0;54f3333e9615:40839 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T02:21:43,107 INFO [regionserver/54f3333e9615:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:21:43,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T02:21:43,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:21:43,108 INFO [M:0;54f3333e9615:40839 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:21:43,108 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T02:21:43,108 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:21:43,109 DEBUG [M:0;54f3333e9615:40839 {}] zookeeper.ZKUtil(347): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T02:21:43,109 WARN [M:0;54f3333e9615:40839 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T02:21:43,109 INFO [M:0;54f3333e9615:40839 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-09T02:21:43,109 INFO [M:0;54f3333e9615:40839 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T02:21:43,109 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:21:43,109 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:43,109 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:43,109 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:21:43,109 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:43,109 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=448.23 KB heapSize=519.78 KB 2024-11-09T02:21:43,127 DEBUG [M:0;54f3333e9615:40839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a09355ce11b9496bb11dd375b50fbd89 is 82, key is hbase:meta,,1/info:regioninfo/1731118705264/Put/seqid=0 2024-11-09T02:21:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741924_1100 (size=5672) 2024-11-09T02:21:43,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:21:43,205 INFO [RS:0;54f3333e9615:45997 {}] regionserver.HRegionServer(1307): Exiting; stopping=54f3333e9615,45997,1731118672827; zookeeper connection closed. 2024-11-09T02:21:43,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45997-0x1018ad21b490001, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:21:43,205 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@557f0077 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@557f0077 2024-11-09T02:21:43,205 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T02:21:43,532 INFO [M:0;54f3333e9615:40839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a09355ce11b9496bb11dd375b50fbd89 2024-11-09T02:21:43,556 DEBUG [M:0;54f3333e9615:40839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bd4d1d3cf8504f8ab09784cdc3f78382 is 830, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1731118707666/Put/seqid=0 2024-11-09T02:21:43,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741925_1101 (size=30260) 2024-11-09T02:21:43,961 INFO [M:0;54f3333e9615:40839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=447.67 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bd4d1d3cf8504f8ab09784cdc3f78382 2024-11-09T02:21:43,966 INFO [M:0;54f3333e9615:40839 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bd4d1d3cf8504f8ab09784cdc3f78382 2024-11-09T02:21:43,981 DEBUG [M:0;54f3333e9615:40839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d620f880a8d543709d519c6ee2a8ee51 is 69, key is 54f3333e9615,45997,1731118672827/rs:state/1731118674585/Put/seqid=0 2024-11-09T02:21:43,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741926_1102 (size=5156) 2024-11-09T02:21:44,386 INFO [M:0;54f3333e9615:40839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d620f880a8d543709d519c6ee2a8ee51 2024-11-09T02:21:44,391 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a09355ce11b9496bb11dd375b50fbd89 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a09355ce11b9496bb11dd375b50fbd89 2024-11-09T02:21:44,396 INFO [M:0;54f3333e9615:40839 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a09355ce11b9496bb11dd375b50fbd89, entries=8, sequenceid=831, filesize=5.5 K 2024-11-09T02:21:44,397 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bd4d1d3cf8504f8ab09784cdc3f78382 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bd4d1d3cf8504f8ab09784cdc3f78382 2024-11-09T02:21:44,402 INFO [M:0;54f3333e9615:40839 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bd4d1d3cf8504f8ab09784cdc3f78382 2024-11-09T02:21:44,402 INFO [M:0;54f3333e9615:40839 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bd4d1d3cf8504f8ab09784cdc3f78382, entries=92, sequenceid=831, filesize=29.6 K 2024-11-09T02:21:44,403 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d620f880a8d543709d519c6ee2a8ee51 as hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d620f880a8d543709d519c6ee2a8ee51 2024-11-09T02:21:44,408 INFO [M:0;54f3333e9615:40839 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40811/user/jenkins/test-data/4cede419-c189-98dd-e4d9-bb076eb90277/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d620f880a8d543709d519c6ee2a8ee51, entries=1, sequenceid=831, filesize=5.0 K 2024-11-09T02:21:44,409 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegion(3040): Finished flush of dataSize ~448.23 KB/458984, heapSize ~519.48 KB/531952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1300ms, sequenceid=831, compaction requested=false 2024-11-09T02:21:44,411 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:21:44,411 DEBUG [M:0;54f3333e9615:40839 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T02:21:44,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32859 is added to blk_1073741830_1006 (size=513918) 2024-11-09T02:21:44,413 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T02:21:44,413 INFO [M:0;54f3333e9615:40839 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-09T02:21:44,414 INFO [M:0;54f3333e9615:40839 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40839 2024-11-09T02:21:44,415 DEBUG [M:0;54f3333e9615:40839 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/54f3333e9615,40839,1731118672115 already deleted, retry=false 2024-11-09T02:21:44,517 INFO [M:0;54f3333e9615:40839 {}] regionserver.HRegionServer(1307): Exiting; stopping=54f3333e9615,40839,1731118672115; zookeeper connection closed. 2024-11-09T02:21:44,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:21:44,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40839-0x1018ad21b490000, quorum=127.0.0.1:57725, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:21:44,522 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@183020ad{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:21:44,524 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5699e1dc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:21:44,524 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:21:44,525 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fc82795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:21:44,525 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72316c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.log.dir/,STOPPED} 2024-11-09T02:21:44,527 WARN [BP-1620340638-172.17.0.2-1731118669061 heartbeating to localhost/127.0.0.1:40811 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T02:21:44,527 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T02:21:44,527 WARN [BP-1620340638-172.17.0.2-1731118669061 heartbeating to localhost/127.0.0.1:40811 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1620340638-172.17.0.2-1731118669061 (Datanode Uuid 62356cfe-2e7b-498f-89e5-84d9dbecc6b0) service to localhost/127.0.0.1:40811 2024-11-09T02:21:44,527 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T02:21:44,529 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/dfs/data/data1/current/BP-1620340638-172.17.0.2-1731118669061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:21:44,529 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/cluster_f97978db-a0e7-70d8-287e-d34da73482d8/dfs/data/data2/current/BP-1620340638-172.17.0.2-1731118669061 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:21:44,529 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T02:21:44,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@50e9ce1d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:21:44,536 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32a5069b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:21:44,536 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:21:44,537 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d739887{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:21:44,537 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@daffc8d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/abc1160f-49d6-391c-a1ce-db0512094023/hadoop.log.dir/,STOPPED} 2024-11-09T02:21:44,545 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-09T02:21:44,579 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down