2024-11-18 22:46:04,001 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-18 22:46:04,038 main DEBUG Took 0.017091 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 22:46:04,038 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 22:46:04,039 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 22:46:04,040 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 22:46:04,041 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,049 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 22:46:04,069 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,072 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,073 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,079 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,080 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,082 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,083 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,086 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,088 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,089 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 22:46:04,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,092 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 22:46:04,101 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 22:46:04,106 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 22:46:04,111 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 22:46:04,114 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 22:46:04,120 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 22:46:04,121 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 22:46:04,137 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 22:46:04,140 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 22:46:04,143 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 22:46:04,143 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 22:46:04,144 main DEBUG createAppenders(={Console}) 2024-11-18 22:46:04,145 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-18 22:46:04,145 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-18 22:46:04,146 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-18 22:46:04,147 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 22:46:04,147 main DEBUG OutputStream closed 2024-11-18 22:46:04,147 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 22:46:04,148 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 22:46:04,148 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-18 22:46:04,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 22:46:04,328 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 22:46:04,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 22:46:04,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 22:46:04,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 22:46:04,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 22:46:04,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 22:46:04,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 22:46:04,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 22:46:04,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 22:46:04,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 22:46:04,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 22:46:04,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 22:46:04,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 22:46:04,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 22:46:04,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 22:46:04,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 22:46:04,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 22:46:04,344 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 22:46:04,345 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-18 22:46:04,345 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 22:46:04,346 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-18T22:46:04,363 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-18 22:46:04,368 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 22:46:04,368 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T22:46:04,757 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70 2024-11-18T22:46:04,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T22:46:04,813 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa, deleteOnExit=true 2024-11-18T22:46:04,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T22:46:04,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/test.cache.data in system properties and HBase conf 2024-11-18T22:46:04,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T22:46:04,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.log.dir in system properties and HBase conf 2024-11-18T22:46:04,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T22:46:04,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T22:46:04,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T22:46:04,917 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T22:46:05,034 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T22:46:05,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T22:46:05,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T22:46:05,040 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T22:46:05,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T22:46:05,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T22:46:05,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T22:46:05,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T22:46:05,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T22:46:05,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T22:46:05,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/nfs.dump.dir in system properties and HBase conf 2024-11-18T22:46:05,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/java.io.tmpdir in system properties and HBase conf 2024-11-18T22:46:05,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T22:46:05,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T22:46:05,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T22:46:06,225 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T22:46:06,303 INFO [Time-limited test {}] log.Log(170): Logging initialized @3477ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T22:46:06,395 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T22:46:06,476 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T22:46:06,497 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T22:46:06,498 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T22:46:06,499 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T22:46:06,513 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T22:46:06,516 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.log.dir/,AVAILABLE} 2024-11-18T22:46:06,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T22:46:06,700 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4df87c21{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/java.io.tmpdir/jetty-localhost-45229-hadoop-hdfs-3_4_1-tests_jar-_-any-483535535144970362/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T22:46:06,709 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:45229} 2024-11-18T22:46:06,710 INFO [Time-limited test {}] server.Server(415): Started @3884ms 2024-11-18T22:46:07,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T22:46:07,586 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T22:46:07,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T22:46:07,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T22:46:07,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T22:46:07,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7aa82d16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.log.dir/,AVAILABLE} 2024-11-18T22:46:07,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23c598c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T22:46:07,736 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f9a1a82{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/java.io.tmpdir/jetty-localhost-36923-hadoop-hdfs-3_4_1-tests_jar-_-any-7232718633646645869/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T22:46:07,737 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@210254cd{HTTP/1.1, (http/1.1)}{localhost:36923} 2024-11-18T22:46:07,737 INFO [Time-limited test {}] server.Server(415): Started @4911ms 2024-11-18T22:46:07,819 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T22:46:09,088 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/data/data2/current/BP-1011012363-172.17.0.2-1731969965710/current, will proceed with Du for space computation calculation, 2024-11-18T22:46:09,088 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/data/data1/current/BP-1011012363-172.17.0.2-1731969965710/current, will proceed with Du for space computation calculation, 2024-11-18T22:46:09,134 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T22:46:09,192 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70 2024-11-18T22:46:09,200 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda895a9a4cd48585 with lease ID 0xe4a244813c7a2561: Processing first storage report for DS-c1f6d64b-babb-4e2e-ada2-9937a51e054e from datanode DatanodeRegistration(127.0.0.1:43817, datanodeUuid=ec45131f-66ab-4b5b-b786-7aeebb4afcc0, infoPort=44475, infoSecurePort=0, ipcPort=36829, storageInfo=lv=-57;cid=testClusterID;nsid=1189418096;c=1731969965710) 2024-11-18T22:46:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda895a9a4cd48585 with lease ID 0xe4a244813c7a2561: from storage DS-c1f6d64b-babb-4e2e-ada2-9937a51e054e node DatanodeRegistration(127.0.0.1:43817, datanodeUuid=ec45131f-66ab-4b5b-b786-7aeebb4afcc0, infoPort=44475, infoSecurePort=0, ipcPort=36829, storageInfo=lv=-57;cid=testClusterID;nsid=1189418096;c=1731969965710), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T22:46:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda895a9a4cd48585 with lease ID 0xe4a244813c7a2561: Processing first storage report for DS-9dd6574a-4dea-409b-9d5d-35f80baec757 from datanode DatanodeRegistration(127.0.0.1:43817, datanodeUuid=ec45131f-66ab-4b5b-b786-7aeebb4afcc0, infoPort=44475, infoSecurePort=0, ipcPort=36829, storageInfo=lv=-57;cid=testClusterID;nsid=1189418096;c=1731969965710) 2024-11-18T22:46:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda895a9a4cd48585 with lease ID 0xe4a244813c7a2561: from storage DS-9dd6574a-4dea-409b-9d5d-35f80baec757 node DatanodeRegistration(127.0.0.1:43817, datanodeUuid=ec45131f-66ab-4b5b-b786-7aeebb4afcc0, infoPort=44475, infoSecurePort=0, ipcPort=36829, storageInfo=lv=-57;cid=testClusterID;nsid=1189418096;c=1731969965710), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T22:46:09,347 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/zookeeper_0, clientPort=60767, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T22:46:09,362 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60767 2024-11-18T22:46:09,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:09,383 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:09,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741825_1001 (size=7) 2024-11-18T22:46:10,082 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 with version=8 2024-11-18T22:46:10,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/hbase-staging 2024-11-18T22:46:10,205 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T22:46:10,461 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3b0ca91ed64a:0 server-side Connection retries=9 2024-11-18T22:46:10,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:10,473 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:10,480 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T22:46:10,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:10,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T22:46:10,659 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T22:46:10,741 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T22:46:10,753 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T22:46:10,758 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T22:46:10,796 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 106937 (auto-detected) 2024-11-18T22:46:10,798 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T22:46:10,824 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36189 2024-11-18T22:46:10,853 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36189 connecting to ZooKeeper ensemble=127.0.0.1:60767 2024-11-18T22:46:10,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361890x0, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T22:46:10,947 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36189-0x1015020684e0000 connected 2024-11-18T22:46:11,200 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:11,202 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:11,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T22:46:11,220 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270, hbase.cluster.distributed=false 2024-11-18T22:46:11,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T22:46:11,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-18T22:46:11,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36189 2024-11-18T22:46:11,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36189 2024-11-18T22:46:11,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-18T22:46:11,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36189 2024-11-18T22:46:11,407 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3b0ca91ed64a:0 server-side Connection retries=9 2024-11-18T22:46:11,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:11,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:11,411 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T22:46:11,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T22:46:11,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T22:46:11,416 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T22:46:11,420 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T22:46:11,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42807 2024-11-18T22:46:11,425 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42807 connecting to ZooKeeper ensemble=127.0.0.1:60767 2024-11-18T22:46:11,427 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:11,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:11,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428070x0, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T22:46:11,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42807-0x1015020684e0001 connected 2024-11-18T22:46:11,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T22:46:11,470 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T22:46:11,479 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T22:46:11,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T22:46:11,492 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T22:46:11,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42807 2024-11-18T22:46:11,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42807 2024-11-18T22:46:11,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42807 2024-11-18T22:46:11,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42807 2024-11-18T22:46:11,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42807 2024-11-18T22:46:11,529 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3b0ca91ed64a:36189 2024-11-18T22:46:11,530 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:11,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T22:46:11,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T22:46:11,551 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:11,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:11,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T22:46:11,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:11,588 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T22:46:11,589 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3b0ca91ed64a,36189,1731969970268 from backup master directory 2024-11-18T22:46:11,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T22:46:11,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:11,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T22:46:11,602 WARN [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T22:46:11,603 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:11,605 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T22:46:11,610 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T22:46:11,671 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/hbase.id] with ID: b53db6ea-93ed-4c93-9b76-cd27c66e0734 2024-11-18T22:46:11,671 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/.tmp/hbase.id 2024-11-18T22:46:11,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741826_1002 (size=42) 2024-11-18T22:46:12,087 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/.tmp/hbase.id]:[hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/hbase.id] 2024-11-18T22:46:12,201 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T22:46:12,208 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T22:46:12,236 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-18T22:46:12,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:12,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:12,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741827_1003 (size=196) 2024-11-18T22:46:12,712 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:46:12,716 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T22:46:12,737 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T22:46:12,741 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T22:46:12,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741828_1004 (size=1189) 2024-11-18T22:46:13,186 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store 2024-11-18T22:46:13,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741829_1005 (size=34) 2024-11-18T22:46:13,615 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T22:46:13,618 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:13,620 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T22:46:13,620 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:46:13,620 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:46:13,622 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T22:46:13,622 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:46:13,622 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:46:13,623 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731969973619Disabling compacts and flushes for region at 1731969973619Disabling writes for close at 1731969973622 (+3 ms)Writing region close event to WAL at 1731969973622Closed at 1731969973622 2024-11-18T22:46:13,626 WARN [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/.initializing 2024-11-18T22:46:13,626 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/WALs/3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:13,636 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T22:46:13,654 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b0ca91ed64a%2C36189%2C1731969970268, suffix=, logDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/WALs/3b0ca91ed64a,36189,1731969970268, archiveDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/oldWALs, maxLogs=10 2024-11-18T22:46:13,680 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/WALs/3b0ca91ed64a,36189,1731969970268/3b0ca91ed64a%2C36189%2C1731969970268.1731969973659, exclude list is [], retry=0 2024-11-18T22:46:13,697 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43817,DS-c1f6d64b-babb-4e2e-ada2-9937a51e054e,DISK] 2024-11-18T22:46:13,700 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T22:46:13,733 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/WALs/3b0ca91ed64a,36189,1731969970268/3b0ca91ed64a%2C36189%2C1731969970268.1731969973659 2024-11-18T22:46:13,734 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44475:44475)] 2024-11-18T22:46:13,734 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:46:13,735 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:13,739 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,742 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T22:46:13,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:13,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:13,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T22:46:13,835 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:13,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:13,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T22:46:13,840 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:13,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:13,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,844 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T22:46:13,844 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:13,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:13,846 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,850 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,851 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,856 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,857 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,861 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T22:46:13,866 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T22:46:13,871 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:46:13,873 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64942032, jitterRate=-0.032288312911987305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T22:46:13,882 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731969973758Initializing all the Stores at 1731969973761 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731969973761Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731969973762 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731969973762Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731969973763 (+1 ms)Cleaning up temporary data from old regions at 1731969973857 (+94 ms)Region opened successfully at 1731969973882 (+25 ms) 2024-11-18T22:46:13,884 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T22:46:13,915 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6dc8d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b0ca91ed64a/172.17.0.2:0 2024-11-18T22:46:13,946 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T22:46:13,957 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T22:46:13,957 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T22:46:13,960 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T22:46:13,961 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T22:46:13,966 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T22:46:13,966 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T22:46:13,992 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T22:46:14,002 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T22:46:14,016 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T22:46:14,019 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T22:46:14,021 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T22:46:14,033 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T22:46:14,036 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T22:46:14,041 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T22:46:14,054 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T22:46:14,056 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T22:46:14,068 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T22:46:14,091 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T22:46:14,100 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T22:46:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T22:46:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T22:46:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,113 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3b0ca91ed64a,36189,1731969970268, sessionid=0x1015020684e0000, setting cluster-up flag (Was=false) 2024-11-18T22:46:14,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,184 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T22:46:14,186 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:14,247 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T22:46:14,249 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:14,255 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T22:46:14,319 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(746): ClusterId : b53db6ea-93ed-4c93-9b76-cd27c66e0734 2024-11-18T22:46:14,322 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T22:46:14,334 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T22:46:14,334 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T22:46:14,339 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T22:46:14,349 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T22:46:14,353 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T22:46:14,354 DEBUG [RS:0;3b0ca91ed64a:42807 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391ef5c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b0ca91ed64a/172.17.0.2:0 2024-11-18T22:46:14,357 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T22:46:14,362 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3b0ca91ed64a,36189,1731969970268 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T22:46:14,370 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3b0ca91ed64a:0, corePoolSize=5, maxPoolSize=5 2024-11-18T22:46:14,370 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3b0ca91ed64a:0, corePoolSize=5, maxPoolSize=5 2024-11-18T22:46:14,370 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3b0ca91ed64a:0, corePoolSize=5, maxPoolSize=5 2024-11-18T22:46:14,370 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3b0ca91ed64a:0, corePoolSize=5, maxPoolSize=5 2024-11-18T22:46:14,371 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3b0ca91ed64a:0, corePoolSize=10, maxPoolSize=10 2024-11-18T22:46:14,371 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,371 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3b0ca91ed64a:0, corePoolSize=2, maxPoolSize=2 2024-11-18T22:46:14,371 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,378 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T22:46:14,379 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T22:46:14,383 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3b0ca91ed64a:42807 2024-11-18T22:46:14,386 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:14,386 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731970004386 2024-11-18T22:46:14,387 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T22:46:14,387 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T22:46:14,387 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T22:46:14,388 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T22:46:14,388 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T22:46:14,390 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T22:46:14,391 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b0ca91ed64a,36189,1731969970268 with port=42807, startcode=1731969971356 2024-11-18T22:46:14,402 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T22:46:14,402 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T22:46:14,402 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T22:46:14,403 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T22:46:14,405 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,413 DEBUG [RS:0;3b0ca91ed64a:42807 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T22:46:14,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741831_1007 (size=1321) 2024-11-18T22:46:14,423 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T22:46:14,424 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:46:14,430 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T22:46:14,432 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T22:46:14,432 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T22:46:14,452 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T22:46:14,453 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T22:46:14,466 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.large.0-1731969974458,5,FailOnTimeoutGroup] 2024-11-18T22:46:14,473 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.small.0-1731969974466,5,FailOnTimeoutGroup] 2024-11-18T22:46:14,473 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,474 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T22:46:14,476 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,477 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741832_1008 (size=32) 2024-11-18T22:46:14,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:14,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T22:46:14,493 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T22:46:14,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:14,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:14,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T22:46:14,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T22:46:14,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:14,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:14,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T22:46:14,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T22:46:14,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:14,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:14,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T22:46:14,515 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T22:46:14,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:14,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:14,517 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T22:46:14,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740 2024-11-18T22:46:14,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740 2024-11-18T22:46:14,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T22:46:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T22:46:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T22:46:14,528 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T22:46:14,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T22:46:14,543 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:46:14,544 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,545 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67416886, jitterRate=0.004589885473251343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T22:46:14,549 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36189 {}] master.ServerManager(517): Registering regionserver=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731969974483Initializing all the Stores at 1731969974486 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731969974486Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731969974486Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731969974486Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731969974486Cleaning up temporary data from old regions at 1731969974527 (+41 ms)Region opened successfully at 1731969974550 (+23 ms) 2024-11-18T22:46:14,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T22:46:14,551 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T22:46:14,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T22:46:14,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T22:46:14,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T22:46:14,556 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T22:46:14,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731969974551Disabling compacts and flushes for region at 1731969974551Disabling writes for close at 1731969974552 (+1 ms)Writing region close event to WAL at 1731969974556 (+4 ms)Closed at 1731969974556 2024-11-18T22:46:14,561 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T22:46:14,562 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T22:46:14,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T22:46:14,576 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:46:14,577 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37885 2024-11-18T22:46:14,577 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T22:46:14,593 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T22:46:14,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T22:46:14,595 DEBUG [RS:0;3b0ca91ed64a:42807 {}] zookeeper.ZKUtil(111): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,596 WARN [RS:0;3b0ca91ed64a:42807 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T22:46:14,596 INFO [RS:0;3b0ca91ed64a:42807 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T22:46:14,596 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,601 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T22:46:14,612 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3b0ca91ed64a,42807,1731969971356] 2024-11-18T22:46:14,652 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T22:46:14,678 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T22:46:14,689 INFO [RS:0;3b0ca91ed64a:42807 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T22:46:14,689 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,693 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T22:46:14,701 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T22:46:14,703 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,704 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,704 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,704 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,704 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,704 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3b0ca91ed64a:0, corePoolSize=2, maxPoolSize=2 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,705 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,706 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3b0ca91ed64a:0, corePoolSize=1, maxPoolSize=1 2024-11-18T22:46:14,706 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3b0ca91ed64a:0, corePoolSize=3, maxPoolSize=3 2024-11-18T22:46:14,706 DEBUG [RS:0;3b0ca91ed64a:42807 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3b0ca91ed64a:0, corePoolSize=3, maxPoolSize=3 2024-11-18T22:46:14,713 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,713 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,713 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,716 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,717 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,717 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,42807,1731969971356-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T22:46:14,753 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:14,767 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T22:46:14,771 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,42807,1731969971356-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,772 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,772 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.Replication(171): 3b0ca91ed64a,42807,1731969971356 started 2024-11-18T22:46:14,816 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:14,817 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1482): Serving as 3b0ca91ed64a,42807,1731969971356, RpcServer on 3b0ca91ed64a/172.17.0.2:42807, sessionid=0x1015020684e0001 2024-11-18T22:46:14,818 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T22:46:14,818 DEBUG [RS:0;3b0ca91ed64a:42807 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,818 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b0ca91ed64a,42807,1731969971356' 2024-11-18T22:46:14,819 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T22:46:14,821 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T22:46:14,822 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T22:46:14,822 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T22:46:14,823 DEBUG [RS:0;3b0ca91ed64a:42807 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:14,823 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b0ca91ed64a,42807,1731969971356' 2024-11-18T22:46:14,823 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T22:46:14,824 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T22:46:14,825 DEBUG [RS:0;3b0ca91ed64a:42807 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T22:46:14,826 INFO [RS:0;3b0ca91ed64a:42807 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T22:46:14,826 INFO [RS:0;3b0ca91ed64a:42807 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T22:46:15,754 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:16,754 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:17,755 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:18,756 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:19,757 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:20,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T22:46:20,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T22:46:20,758 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:21,758 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:22,759 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:23,760 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:24,762 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:25,763 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:26,763 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:27,764 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:28,765 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:29,766 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:30,767 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:31,774 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:32,775 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:33,778 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:34,786 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:35,793 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:36,795 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:37,796 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:38,798 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:39,188 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T22:46:39,799 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:40,800 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:41,801 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:42,801 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:43,802 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:44,803 WARN [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-18T22:46:44,834 INFO [RS:0;3b0ca91ed64a:42807 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T22:46:44,851 INFO [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b0ca91ed64a%2C42807%2C1731969971356, suffix=, logDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356, archiveDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/oldWALs, maxLogs=32 2024-11-18T22:46:44,883 DEBUG [RS:0;3b0ca91ed64a:42807 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356/3b0ca91ed64a%2C42807%2C1731969971356.1731970004857, exclude list is [], retry=0 2024-11-18T22:46:44,905 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43817,DS-c1f6d64b-babb-4e2e-ada2-9937a51e054e,DISK] 2024-11-18T22:46:44,921 INFO [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356/3b0ca91ed64a%2C42807%2C1731969971356.1731970004857 2024-11-18T22:46:44,922 DEBUG [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44475:44475)] 2024-11-18T22:46:45,056 DEBUG [3b0ca91ed64a:36189 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T22:46:45,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:45,091 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b0ca91ed64a,42807,1731969971356, state=OPENING 2024-11-18T22:46:45,108 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T22:46:45,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:45,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:46:45,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T22:46:45,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T22:46:45,123 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T22:46:45,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:46:45,334 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T22:46:45,364 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33719, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T22:46:45,393 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T22:46:45,394 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T22:46:45,395 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T22:46:45,403 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b0ca91ed64a%2C42807%2C1731969971356.meta, suffix=.meta, logDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356, archiveDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/oldWALs, maxLogs=32 2024-11-18T22:46:45,424 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356/3b0ca91ed64a%2C42807%2C1731969971356.meta.1731970005406.meta, exclude list is [], retry=0 2024-11-18T22:46:45,433 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43817,DS-c1f6d64b-babb-4e2e-ada2-9937a51e054e,DISK] 2024-11-18T22:46:45,456 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356/3b0ca91ed64a%2C42807%2C1731969971356.meta.1731970005406.meta 2024-11-18T22:46:45,464 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44475:44475)] 2024-11-18T22:46:45,465 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:46:45,470 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T22:46:45,480 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T22:46:45,488 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T22:46:45,495 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T22:46:45,496 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:45,496 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T22:46:45,497 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T22:46:45,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T22:46:45,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T22:46:45,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:45,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:45,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T22:46:45,513 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T22:46:45,514 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:45,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:45,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T22:46:45,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T22:46:45,520 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:45,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:45,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T22:46:45,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T22:46:45,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:45,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T22:46:45,538 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T22:46:45,544 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740 2024-11-18T22:46:45,551 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740 2024-11-18T22:46:45,554 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T22:46:45,554 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T22:46:45,555 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T22:46:45,569 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T22:46:45,572 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68421245, jitterRate=0.01955600082874298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T22:46:45,573 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T22:46:45,575 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731970005498Writing region info on filesystem at 1731970005498Initializing all the Stores at 1731970005500 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731970005500Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731970005500Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970005500Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731970005500Cleaning up temporary data from old regions at 1731970005554 (+54 ms)Running coprocessor post-open hooks at 1731970005573 (+19 ms)Region opened successfully at 1731970005575 (+2 ms) 2024-11-18T22:46:45,586 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731970005321 2024-11-18T22:46:45,602 DEBUG [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T22:46:45,603 INFO [RS_OPEN_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T22:46:45,606 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:45,609 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b0ca91ed64a,42807,1731969971356, state=OPEN 2024-11-18T22:46:45,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T22:46:45,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T22:46:45,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T22:46:45,698 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:45,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T22:46:45,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T22:46:45,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b0ca91ed64a,42807,1731969971356 in 573 msec 2024-11-18T22:46:45,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T22:46:45,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 31.1400 sec 2024-11-18T22:46:45,734 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T22:46:45,734 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T22:46:45,758 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T22:46:45,760 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b0ca91ed64a,42807,1731969971356, seqNum=-1] 2024-11-18T22:46:45,791 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T22:46:45,794 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42027, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T22:46:45,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.5380 sec 2024-11-18T22:46:45,831 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731970005831, completionTime=-1 2024-11-18T22:46:45,835 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T22:46:45,836 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T22:46:45,881 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T22:46:45,881 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731970065881 2024-11-18T22:46:45,881 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731970125881 2024-11-18T22:46:45,881 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 45 msec 2024-11-18T22:46:45,884 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,885 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,885 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,887 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3b0ca91ed64a:36189, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,887 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,888 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:45,899 DEBUG [master/3b0ca91ed64a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T22:46:45,928 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 34.325sec 2024-11-18T22:46:45,930 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T22:46:45,932 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T22:46:45,934 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T22:46:45,935 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T22:46:45,935 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T22:46:45,936 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T22:46:45,937 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T22:46:45,957 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T22:46:45,958 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T22:46:45,958 INFO [master/3b0ca91ed64a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b0ca91ed64a,36189,1731969970268-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T22:46:46,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557c9aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,004 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T22:46:46,004 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T22:46:46,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b0ca91ed64a,36189,-1 for getting cluster id 2024-11-18T22:46:46,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T22:46:46,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b53db6ea-93ed-4c93-9b76-cd27c66e0734' 2024-11-18T22:46:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T22:46:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b53db6ea-93ed-4c93-9b76-cd27c66e0734" 2024-11-18T22:46:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d401c59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b0ca91ed64a,36189,-1] 2024-11-18T22:46:46,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T22:46:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:46:46,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T22:46:46,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f6d83e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T22:46:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b0ca91ed64a,42807,1731969971356, seqNum=-1] 2024-11-18T22:46:46,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T22:46:46,065 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T22:46:46,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:46,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438beceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b0ca91ed64a,36189,-1 for getting cluster id 2024-11-18T22:46:46,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T22:46:46,113 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b53db6ea-93ed-4c93-9b76-cd27c66e0734' 2024-11-18T22:46:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T22:46:46,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b53db6ea-93ed-4c93-9b76-cd27c66e0734" 2024-11-18T22:46:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44627131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b0ca91ed64a,36189,-1] 2024-11-18T22:46:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T22:46:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:46:46,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T22:46:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9e7cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T22:46:46,155 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=213, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6688 2024-11-18T22:46:46,163 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T22:46:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b0ca91ed64a,36189,1731969970268 2024-11-18T22:46:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c9d4e99 2024-11-18T22:46:46,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T22:46:46,221 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T22:46:46,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:46:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-18T22:46:46,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:46:46,272 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:46,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-18T22:46:46,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:46:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T22:46:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741835_1011 (size=428) 2024-11-18T22:46:46,373 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 657ca8bc4f923a7997087ddc42f1eae1, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:46:46,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741836_1012 (size=79) 2024-11-18T22:46:46,455 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:46,455 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 657ca8bc4f923a7997087ddc42f1eae1, disabling compactions & flushes 2024-11-18T22:46:46,455 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,456 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,456 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. after waiting 0 ms 2024-11-18T22:46:46,456 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,456 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,456 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 657ca8bc4f923a7997087ddc42f1eae1: Waiting for close lock at 1731970006455Disabling compacts and flushes for region at 1731970006455Disabling writes for close at 1731970006456 (+1 ms)Writing region close event to WAL at 1731970006456Closed at 1731970006456 2024-11-18T22:46:46,459 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:46:46,466 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731970006459"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970006459"}]},"ts":"1731970006459"} 2024-11-18T22:46:46,473 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:46:46,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:46:46,478 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970006475"}]},"ts":"1731970006475"} 2024-11-18T22:46:46,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-18T22:46:46,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=657ca8bc4f923a7997087ddc42f1eae1, ASSIGN}] 2024-11-18T22:46:46,493 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=657ca8bc4f923a7997087ddc42f1eae1, ASSIGN 2024-11-18T22:46:46,501 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=657ca8bc4f923a7997087ddc42f1eae1, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:46:46,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=657ca8bc4f923a7997087ddc42f1eae1, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:46,661 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=657ca8bc4f923a7997087ddc42f1eae1, ASSIGN because future has completed 2024-11-18T22:46:46,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 657ca8bc4f923a7997087ddc42f1eae1, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:46:46,829 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,829 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 657ca8bc4f923a7997087ddc42f1eae1, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:46:46,830 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,830 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:46,830 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,830 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,836 INFO [StoreOpener-657ca8bc4f923a7997087ddc42f1eae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,840 INFO [StoreOpener-657ca8bc4f923a7997087ddc42f1eae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 657ca8bc4f923a7997087ddc42f1eae1 columnFamilyName testFamily 2024-11-18T22:46:46,840 DEBUG [StoreOpener-657ca8bc4f923a7997087ddc42f1eae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:46,842 INFO [StoreOpener-657ca8bc4f923a7997087ddc42f1eae1-1 {}] regionserver.HStore(327): Store=657ca8bc4f923a7997087ddc42f1eae1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:46,843 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,845 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,846 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,848 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,848 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,857 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,862 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:46:46,863 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 657ca8bc4f923a7997087ddc42f1eae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68568048, jitterRate=0.0217435359954834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:46:46,863 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:46:46,865 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 657ca8bc4f923a7997087ddc42f1eae1: Running coprocessor pre-open hook at 1731970006830Writing region info on filesystem at 1731970006830Initializing all the Stores at 1731970006834 (+4 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970006834Cleaning up temporary data from old regions at 1731970006848 (+14 ms)Running coprocessor post-open hooks at 1731970006864 (+16 ms)Region opened successfully at 1731970006865 (+1 ms) 2024-11-18T22:46:46,868 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1., pid=6, masterSystemTime=1731970006820 2024-11-18T22:46:46,879 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=657ca8bc4f923a7997087ddc42f1eae1, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:46,881 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,881 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:46:46,886 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 657ca8bc4f923a7997087ddc42f1eae1, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:46:46,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T22:46:46,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 657ca8bc4f923a7997087ddc42f1eae1, server=3b0ca91ed64a,42807,1731969971356 in 229 msec 2024-11-18T22:46:46,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T22:46:46,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=657ca8bc4f923a7997087ddc42f1eae1, ASSIGN in 409 msec 2024-11-18T22:46:46,907 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:46:46,908 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970006908"}]},"ts":"1731970006908"} 2024-11-18T22:46:46,913 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-18T22:46:46,915 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:46:46,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 678 msec 2024-11-18T22:46:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T22:46:47,304 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-18T22:46:47,304 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-18T22:46:47,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:47,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-18T22:46:47,314 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:47,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-18T22:46:47,318 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T22:46:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b0ca91ed64a,42807,1731969971356, seqNum=-1] 2024-11-18T22:46:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T22:46:47,324 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T22:46:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:46:47,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:46:47,367 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-18T22:46:47,375 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-18T22:46:47,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-18T22:46:47,386 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-18T22:46:47,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-581467765047558136 2024-11-18T22:46:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970010386, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:46:47,425 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-18T22:46:47,428 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-18T22:46:47,430 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:46:47,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:46:47,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-581467765047558134 2024-11-18T22:46:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970010433, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:46:47,443 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:46:47,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-581467765047558133 2024-11-18T22:46:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970010443, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:46:47,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:46:47,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-581467765047558132 2024-11-18T22:46:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970010452, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:46:47,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:46:47,478 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=219 (was 213) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-770510147_22 at /127.0.0.1:54250 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6536 (was 6688) 2024-11-18T22:46:47,495 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=219, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6533 2024-11-18T22:46:47,505 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:46:47,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-18T22:46:47,532 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:46:47,533 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:47,533 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-18T22:46:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T22:46:47,538 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:46:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741837_1013 (size=414) 2024-11-18T22:46:47,661 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 80d83b5a1b5a68e035c174507dbe0593, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:46:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741838_1014 (size=65) 2024-11-18T22:46:47,732 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:47,732 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 80d83b5a1b5a68e035c174507dbe0593, disabling compactions & flushes 2024-11-18T22:46:47,733 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:47,733 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:47,733 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. after waiting 0 ms 2024-11-18T22:46:47,733 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:47,733 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:47,733 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 80d83b5a1b5a68e035c174507dbe0593: Waiting for close lock at 1731970007732Disabling compacts and flushes for region at 1731970007732Disabling writes for close at 1731970007733 (+1 ms)Writing region close event to WAL at 1731970007733Closed at 1731970007733 2024-11-18T22:46:47,741 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:46:47,742 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731970007741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970007741"}]},"ts":"1731970007741"} 2024-11-18T22:46:47,768 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:46:47,776 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:46:47,778 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970007777"}]},"ts":"1731970007777"} 2024-11-18T22:46:47,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-18T22:46:47,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=80d83b5a1b5a68e035c174507dbe0593, ASSIGN}] 2024-11-18T22:46:47,804 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=80d83b5a1b5a68e035c174507dbe0593, ASSIGN 2024-11-18T22:46:47,808 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=80d83b5a1b5a68e035c174507dbe0593, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:46:47,960 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=80d83b5a1b5a68e035c174507dbe0593, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:47,965 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=80d83b5a1b5a68e035c174507dbe0593, ASSIGN because future has completed 2024-11-18T22:46:47,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80d83b5a1b5a68e035c174507dbe0593, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:46:48,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:48,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 80d83b5a1b5a68e035c174507dbe0593, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:46:48,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:48,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,135 INFO [StoreOpener-80d83b5a1b5a68e035c174507dbe0593-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,138 INFO [StoreOpener-80d83b5a1b5a68e035c174507dbe0593-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80d83b5a1b5a68e035c174507dbe0593 columnFamilyName testFamily 2024-11-18T22:46:48,138 DEBUG [StoreOpener-80d83b5a1b5a68e035c174507dbe0593-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:48,139 INFO [StoreOpener-80d83b5a1b5a68e035c174507dbe0593-1 {}] regionserver.HStore(327): Store=80d83b5a1b5a68e035c174507dbe0593/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:48,139 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,145 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,145 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,150 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,157 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:46:48,158 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 80d83b5a1b5a68e035c174507dbe0593; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66568505, jitterRate=-0.008051976561546326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:46:48,158 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:46:48,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 80d83b5a1b5a68e035c174507dbe0593: Running coprocessor pre-open hook at 1731970008129Writing region info on filesystem at 1731970008129Initializing all the Stores at 1731970008134 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970008134Cleaning up temporary data from old regions at 1731970008145 (+11 ms)Running coprocessor post-open hooks at 1731970008158 (+13 ms)Region opened successfully at 1731970008160 (+2 ms) 2024-11-18T22:46:48,176 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., pid=9, masterSystemTime=1731970008121 2024-11-18T22:46:48,183 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:48,183 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:46:48,186 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=80d83b5a1b5a68e035c174507dbe0593, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:48,198 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80d83b5a1b5a68e035c174507dbe0593, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:46:48,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T22:46:48,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 80d83b5a1b5a68e035c174507dbe0593, server=3b0ca91ed64a,42807,1731969971356 in 236 msec 2024-11-18T22:46:48,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T22:46:48,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=80d83b5a1b5a68e035c174507dbe0593, ASSIGN in 420 msec 2024-11-18T22:46:48,228 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:46:48,229 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970008228"}]},"ts":"1731970008228"} 2024-11-18T22:46:48,233 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-18T22:46:48,237 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:46:48,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 731 msec 2024-11-18T22:46:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T22:46:48,553 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-18T22:46:48,553 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-18T22:46:48,553 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:48,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-18T22:46:48,558 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:48,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-18T22:46:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:46:48,577 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:46:48,578 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-18T22:46:48,582 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T22:46:48,584 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-18T22:46:51,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:46:51,074 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-18T22:46:51,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T22:46:51,077 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-18T22:46:51,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T22:46:51,079 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T22:46:51,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T22:46:51,080 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T22:46:51,081 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-18T22:46:51,081 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T22:46:51,082 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-18T22:46:51,082 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-18T22:46:54,599 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-18T22:46:54,600 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-18T22:46:54,600 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:46:54,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=222 (was 219) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=741 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5917 (was 6533) 2024-11-18T22:46:54,623 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=741, ProcessCount=11, AvailableMemoryMB=5916 2024-11-18T22:46:54,625 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:46:54,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-18T22:46:54,629 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:46:54,629 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:54,629 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-18T22:46:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-18T22:46:54,631 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:46:54,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741839_1015 (size=412) 2024-11-18T22:46:55,043 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4dda8cbd2ec5ba2edae339bcd4398d1a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:46:55,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741840_1016 (size=63) 2024-11-18T22:46:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-18T22:46:55,454 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:55,454 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4dda8cbd2ec5ba2edae339bcd4398d1a, disabling compactions & flushes 2024-11-18T22:46:55,454 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,454 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,455 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. after waiting 1 ms 2024-11-18T22:46:55,455 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,455 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,455 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4dda8cbd2ec5ba2edae339bcd4398d1a: Waiting for close lock at 1731970015454Disabling compacts and flushes for region at 1731970015454Disabling writes for close at 1731970015455 (+1 ms)Writing region close event to WAL at 1731970015455Closed at 1731970015455 2024-11-18T22:46:55,457 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:46:55,458 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731970015457"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970015457"}]},"ts":"1731970015457"} 2024-11-18T22:46:55,461 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:46:55,463 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:46:55,464 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970015463"}]},"ts":"1731970015463"} 2024-11-18T22:46:55,467 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-18T22:46:55,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=4dda8cbd2ec5ba2edae339bcd4398d1a, ASSIGN}] 2024-11-18T22:46:55,470 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=4dda8cbd2ec5ba2edae339bcd4398d1a, ASSIGN 2024-11-18T22:46:55,472 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=4dda8cbd2ec5ba2edae339bcd4398d1a, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:46:55,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4dda8cbd2ec5ba2edae339bcd4398d1a, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:55,627 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=4dda8cbd2ec5ba2edae339bcd4398d1a, ASSIGN because future has completed 2024-11-18T22:46:55,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dda8cbd2ec5ba2edae339bcd4398d1a, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:46:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-18T22:46:55,789 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,789 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 4dda8cbd2ec5ba2edae339bcd4398d1a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:46:55,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:46:55,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,794 INFO [StoreOpener-4dda8cbd2ec5ba2edae339bcd4398d1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,798 INFO [StoreOpener-4dda8cbd2ec5ba2edae339bcd4398d1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dda8cbd2ec5ba2edae339bcd4398d1a columnFamilyName testFamily 2024-11-18T22:46:55,798 DEBUG [StoreOpener-4dda8cbd2ec5ba2edae339bcd4398d1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:46:55,799 INFO [StoreOpener-4dda8cbd2ec5ba2edae339bcd4398d1a-1 {}] regionserver.HStore(327): Store=4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:46:55,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,802 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,808 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:46:55,814 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 4dda8cbd2ec5ba2edae339bcd4398d1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66331847, jitterRate=-0.011578455567359924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:46:55,814 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:46:55,815 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 4dda8cbd2ec5ba2edae339bcd4398d1a: Running coprocessor pre-open hook at 1731970015791Writing region info on filesystem at 1731970015791Initializing all the Stores at 1731970015793 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970015793Cleaning up temporary data from old regions at 1731970015803 (+10 ms)Running coprocessor post-open hooks at 1731970015814 (+11 ms)Region opened successfully at 1731970015815 (+1 ms) 2024-11-18T22:46:55,816 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a., pid=12, masterSystemTime=1731970015783 2024-11-18T22:46:55,820 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,820 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:46:55,821 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4dda8cbd2ec5ba2edae339bcd4398d1a, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:46:55,826 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4dda8cbd2ec5ba2edae339bcd4398d1a, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:46:55,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T22:46:55,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4dda8cbd2ec5ba2edae339bcd4398d1a, server=3b0ca91ed64a,42807,1731969971356 in 201 msec 2024-11-18T22:46:55,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T22:46:55,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=4dda8cbd2ec5ba2edae339bcd4398d1a, ASSIGN in 365 msec 2024-11-18T22:46:55,840 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:46:55,840 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970015840"}]},"ts":"1731970015840"} 2024-11-18T22:46:55,844 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-18T22:46:55,846 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:46:55,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2210 sec 2024-11-18T22:46:57,079 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-18T22:46:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-18T22:46:57,663 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-18T22:46:57,663 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-18T22:46:57,663 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:57,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-18T22:46:57,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:46:57,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-18T22:46:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:46:57,687 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:46:57,687 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-18T22:46:57,691 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T22:46:57,694 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T22:46:57,696 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T22:46:57,698 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-18T22:47:00,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-18T22:47:00,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:01,293 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:03,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558123' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T22:47:03,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51866 deadline: 1731970029697, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558123' does not exist 2024-11-18T22:47:03,704 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3b0ca91ed64a,42807,1731969971356 for closing scanner id = -581467765047558123 for 4dda8cbd2ec5ba2edae339bcd4398d1a of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558123' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T22:47:03,707 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=224 (was 222) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=681 (was 741), ProcessCount=11 (was 11), AvailableMemoryMB=5739 (was 5916) 2024-11-18T22:47:03,715 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=681, ProcessCount=11, AvailableMemoryMB=5738 2024-11-18T22:47:03,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-18T22:47:03,721 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:03,721 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:03,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-18T22:47:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T22:47:03,723 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:03,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741841_1017 (size=423) 2024-11-18T22:47:04,139 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fbab814c6ccb19ba7363c5ee6d75166e, NAME => 'testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:04,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741842_1018 (size=74) 2024-11-18T22:47:04,155 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:04,155 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing fbab814c6ccb19ba7363c5ee6d75166e, disabling compactions & flushes 2024-11-18T22:47:04,155 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. after waiting 0 ms 2024-11-18T22:47:04,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,156 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,156 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for fbab814c6ccb19ba7363c5ee6d75166e: Waiting for close lock at 1731970024155Disabling compacts and flushes for region at 1731970024155Disabling writes for close at 1731970024156 (+1 ms)Writing region close event to WAL at 1731970024156Closed at 1731970024156 2024-11-18T22:47:04,158 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:04,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731970024158"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970024158"}]},"ts":"1731970024158"} 2024-11-18T22:47:04,163 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:04,165 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:04,165 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970024165"}]},"ts":"1731970024165"} 2024-11-18T22:47:04,169 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-18T22:47:04,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=fbab814c6ccb19ba7363c5ee6d75166e, ASSIGN}] 2024-11-18T22:47:04,172 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=fbab814c6ccb19ba7363c5ee6d75166e, ASSIGN 2024-11-18T22:47:04,173 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=fbab814c6ccb19ba7363c5ee6d75166e, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-18T22:47:04,324 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fbab814c6ccb19ba7363c5ee6d75166e, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:04,327 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=fbab814c6ccb19ba7363c5ee6d75166e, ASSIGN because future has completed 2024-11-18T22:47:04,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbab814c6ccb19ba7363c5ee6d75166e, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:04,489 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,490 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => fbab814c6ccb19ba7363c5ee6d75166e, NAME => 'testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:04,490 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,490 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:04,490 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,490 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,500 INFO [StoreOpener-fbab814c6ccb19ba7363c5ee6d75166e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,503 INFO [StoreOpener-fbab814c6ccb19ba7363c5ee6d75166e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fbab814c6ccb19ba7363c5ee6d75166e columnFamilyName testFamily 2024-11-18T22:47:04,504 DEBUG [StoreOpener-fbab814c6ccb19ba7363c5ee6d75166e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:04,504 INFO [StoreOpener-fbab814c6ccb19ba7363c5ee6d75166e-1 {}] regionserver.HStore(327): Store=fbab814c6ccb19ba7363c5ee6d75166e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:04,504 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,506 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,506 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,507 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,507 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,509 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,513 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:04,513 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened fbab814c6ccb19ba7363c5ee6d75166e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63851033, jitterRate=-0.048545464873313904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:04,514 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:04,514 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for fbab814c6ccb19ba7363c5ee6d75166e: Running coprocessor pre-open hook at 1731970024490Writing region info on filesystem at 1731970024490Initializing all the Stores at 1731970024493 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970024493Cleaning up temporary data from old regions at 1731970024507 (+14 ms)Running coprocessor post-open hooks at 1731970024514 (+7 ms)Region opened successfully at 1731970024514 2024-11-18T22:47:04,516 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e., pid=15, masterSystemTime=1731970024482 2024-11-18T22:47:04,519 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,519 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:04,520 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=fbab814c6ccb19ba7363c5ee6d75166e, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:04,524 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure fbab814c6ccb19ba7363c5ee6d75166e, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:04,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-18T22:47:04,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure fbab814c6ccb19ba7363c5ee6d75166e, server=3b0ca91ed64a,42807,1731969971356 in 201 msec 2024-11-18T22:47:04,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T22:47:04,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=fbab814c6ccb19ba7363c5ee6d75166e, ASSIGN in 363 msec 2024-11-18T22:47:04,540 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:04,540 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970024540"}]},"ts":"1731970024540"} 2024-11-18T22:47:04,543 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-18T22:47:04,545 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:04,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 827 msec 2024-11-18T22:47:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T22:47:04,733 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-18T22:47:04,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-18T22:47:04,733 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:04,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-18T22:47:04,741 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:04,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-18T22:47:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:04,774 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:04,774 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-18T22:47:04,781 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-18T22:47:04,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-18T22:47:04,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-18T22:47:04,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-581467765047558119 2024-11-18T22:47:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970027789, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:47:04,800 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-18T22:47:04,802 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-18T22:47:04,803 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:47:04,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:47:04,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-581467765047558117 2024-11-18T22:47:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970027806, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:47:04,812 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:47:04,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-581467765047558116 2024-11-18T22:47:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970027811, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:47:04,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:47:04,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-581467765047558115 2024-11-18T22:47:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51866 deadline: 1731970027817, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T22:47:04,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T22:47:04,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=227 (was 224) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-770510147_22 at /127.0.0.1:35580 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=659 (was 681), ProcessCount=11 (was 11), AvailableMemoryMB=5693 (was 5738) 2024-11-18T22:47:04,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=227, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=659, ProcessCount=11, AvailableMemoryMB=5693 2024-11-18T22:47:04,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-18T22:47:04,852 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:04,853 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:04,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-18T22:47:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-18T22:47:04,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741843_1019 (size=418) 2024-11-18T22:47:05,285 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c51f38b82080c8d15458e3951b949b07, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:05,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741844_1020 (size=69) 2024-11-18T22:47:05,708 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:05,708 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing c51f38b82080c8d15458e3951b949b07, disabling compactions & flushes 2024-11-18T22:47:05,708 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:05,708 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:05,708 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. after waiting 0 ms 2024-11-18T22:47:05,708 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:05,709 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:05,709 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for c51f38b82080c8d15458e3951b949b07: Waiting for close lock at 1731970025708Disabling compacts and flushes for region at 1731970025708Disabling writes for close at 1731970025708Writing region close event to WAL at 1731970025709 (+1 ms)Closed at 1731970025709 2024-11-18T22:47:05,710 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:05,711 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731970025710"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970025710"}]},"ts":"1731970025710"} 2024-11-18T22:47:05,714 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:05,716 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:05,716 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970025716"}]},"ts":"1731970025716"} 2024-11-18T22:47:05,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-18T22:47:05,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=c51f38b82080c8d15458e3951b949b07, ASSIGN}] 2024-11-18T22:47:05,721 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=c51f38b82080c8d15458e3951b949b07, ASSIGN 2024-11-18T22:47:05,722 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=c51f38b82080c8d15458e3951b949b07, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-18T22:47:05,873 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c51f38b82080c8d15458e3951b949b07, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:05,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=c51f38b82080c8d15458e3951b949b07, ASSIGN because future has completed 2024-11-18T22:47:05,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c51f38b82080c8d15458e3951b949b07, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:06,039 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:06,040 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => c51f38b82080c8d15458e3951b949b07, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:06,041 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,041 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:06,041 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,041 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,043 INFO [StoreOpener-c51f38b82080c8d15458e3951b949b07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,046 INFO [StoreOpener-c51f38b82080c8d15458e3951b949b07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c51f38b82080c8d15458e3951b949b07 columnFamilyName testFamily 2024-11-18T22:47:06,046 DEBUG [StoreOpener-c51f38b82080c8d15458e3951b949b07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:06,046 INFO [StoreOpener-c51f38b82080c8d15458e3951b949b07-1 {}] regionserver.HStore(327): Store=c51f38b82080c8d15458e3951b949b07/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:06,047 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,048 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,048 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,049 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,049 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,051 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,055 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:06,056 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened c51f38b82080c8d15458e3951b949b07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71818891, jitterRate=0.0701848715543747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:06,057 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:06,057 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for c51f38b82080c8d15458e3951b949b07: Running coprocessor pre-open hook at 1731970026041Writing region info on filesystem at 1731970026041Initializing all the Stores at 1731970026043 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970026043Cleaning up temporary data from old regions at 1731970026049 (+6 ms)Running coprocessor post-open hooks at 1731970026057 (+8 ms)Region opened successfully at 1731970026057 2024-11-18T22:47:06,059 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., pid=18, masterSystemTime=1731970026034 2024-11-18T22:47:06,063 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:06,063 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:06,064 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c51f38b82080c8d15458e3951b949b07, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:06,067 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c51f38b82080c8d15458e3951b949b07, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:06,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T22:47:06,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure c51f38b82080c8d15458e3951b949b07, server=3b0ca91ed64a,42807,1731969971356 in 191 msec 2024-11-18T22:47:06,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-18T22:47:06,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=c51f38b82080c8d15458e3951b949b07, ASSIGN in 356 msec 2024-11-18T22:47:06,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:06,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970026080"}]},"ts":"1731970026080"} 2024-11-18T22:47:06,083 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-18T22:47:06,084 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:06,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2350 sec 2024-11-18T22:47:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-18T22:47:07,882 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-18T22:47:07,883 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-18T22:47:07,883 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:07,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-18T22:47:07,889 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:07,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-18T22:47:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:07,908 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:07,909 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-18T22:47:07,913 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T22:47:07,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T22:47:08,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-18T22:47:08,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-18T22:47:08,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:47:09,082 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-18T22:47:09,083 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-18T22:47:09,188 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T22:47:09,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-18T22:47:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:09,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T22:47:09,992 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:10,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-18T22:47:10,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:10,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-18T22:47:10,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:10,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-18T22:47:10,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-18T22:47:10,917 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:47:10,930 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=221 (was 227), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 659) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6244 (was 5693) - AvailableMemoryMB LEAK? - 2024-11-18T22:47:10,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=221, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=6243 2024-11-18T22:47:10,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-18T22:47:10,952 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:10,952 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:10,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-18T22:47:10,955 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T22:47:10,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741845_1021 (size=416) 2024-11-18T22:47:10,992 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd810abb8d573a852f696b7ab2ca83cc, NAME => 'testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:11,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741846_1022 (size=67) 2024-11-18T22:47:11,437 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:11,437 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing bd810abb8d573a852f696b7ab2ca83cc, disabling compactions & flushes 2024-11-18T22:47:11,437 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,437 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,437 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. after waiting 0 ms 2024-11-18T22:47:11,437 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,438 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,438 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd810abb8d573a852f696b7ab2ca83cc: Waiting for close lock at 1731970031437Disabling compacts and flushes for region at 1731970031437Disabling writes for close at 1731970031437Writing region close event to WAL at 1731970031437Closed at 1731970031437 2024-11-18T22:47:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-18T22:47:11,441 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:11,442 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731970031441"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970031441"}]},"ts":"1731970031441"} 2024-11-18T22:47:11,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:11,448 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:11,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970031448"}]},"ts":"1731970031448"} 2024-11-18T22:47:11,453 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-18T22:47:11,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bd810abb8d573a852f696b7ab2ca83cc, ASSIGN}] 2024-11-18T22:47:11,457 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bd810abb8d573a852f696b7ab2ca83cc, ASSIGN 2024-11-18T22:47:11,459 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bd810abb8d573a852f696b7ab2ca83cc, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:11,610 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bd810abb8d573a852f696b7ab2ca83cc, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:11,614 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bd810abb8d573a852f696b7ab2ca83cc, ASSIGN because future has completed 2024-11-18T22:47:11,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd810abb8d573a852f696b7ab2ca83cc, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:11,775 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,775 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => bd810abb8d573a852f696b7ab2ca83cc, NAME => 'testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:11,776 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,776 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:11,776 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,776 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,778 INFO [StoreOpener-bd810abb8d573a852f696b7ab2ca83cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,785 INFO [StoreOpener-bd810abb8d573a852f696b7ab2ca83cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd810abb8d573a852f696b7ab2ca83cc columnFamilyName testFamily 2024-11-18T22:47:11,785 DEBUG [StoreOpener-bd810abb8d573a852f696b7ab2ca83cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:11,786 INFO [StoreOpener-bd810abb8d573a852f696b7ab2ca83cc-1 {}] regionserver.HStore(327): Store=bd810abb8d573a852f696b7ab2ca83cc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:11,786 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,788 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,788 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,789 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,789 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,792 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,795 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:11,796 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened bd810abb8d573a852f696b7ab2ca83cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69786555, jitterRate=0.039900705218315125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:11,796 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:11,797 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for bd810abb8d573a852f696b7ab2ca83cc: Running coprocessor pre-open hook at 1731970031776Writing region info on filesystem at 1731970031776Initializing all the Stores at 1731970031778 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970031778Cleaning up temporary data from old regions at 1731970031789 (+11 ms)Running coprocessor post-open hooks at 1731970031796 (+7 ms)Region opened successfully at 1731970031797 (+1 ms) 2024-11-18T22:47:11,799 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc., pid=21, masterSystemTime=1731970031769 2024-11-18T22:47:11,802 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,802 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:11,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bd810abb8d573a852f696b7ab2ca83cc, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:11,807 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd810abb8d573a852f696b7ab2ca83cc, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:11,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-18T22:47:11,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure bd810abb8d573a852f696b7ab2ca83cc, server=3b0ca91ed64a,42807,1731969971356 in 195 msec 2024-11-18T22:47:11,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T22:47:11,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bd810abb8d573a852f696b7ab2ca83cc, ASSIGN in 359 msec 2024-11-18T22:47:11,819 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:11,819 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970031819"}]},"ts":"1731970031819"} 2024-11-18T22:47:11,823 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-18T22:47:11,825 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:11,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 879 msec 2024-11-18T22:47:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T22:47:11,973 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-18T22:47:11,973 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-18T22:47:11,974 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:11,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-18T22:47:11,980 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:11,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-18T22:47:11,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:12,001 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:12,002 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-18T22:47:12,006 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T22:47:12,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T22:47:12,012 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T22:47:12,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-18T22:47:15,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558104' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T22:47:15,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51866 deadline: 1731970036020, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558104' does not exist 2024-11-18T22:47:15,026 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3b0ca91ed64a,42807,1731969971356 for closing scanner id = -581467765047558104 for bd810abb8d573a852f696b7ab2ca83cc of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558104' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T22:47:15,032 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 221) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-770510147_22 at /127.0.0.1:35630 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=742 (was 694) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6970 (was 6243) - AvailableMemoryMB LEAK? - 2024-11-18T22:47:15,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=742, ProcessCount=11, AvailableMemoryMB=6969 2024-11-18T22:47:15,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-18T22:47:15,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:15,059 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:15,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-18T22:47:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-18T22:47:15,062 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:15,084 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-18T22:47:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741847_1023 (size=411) 2024-11-18T22:47:15,117 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ece1749e1d126e57e47de401d52563ef, NAME => 'testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:15,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741848_1024 (size=62) 2024-11-18T22:47:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-18T22:47:15,536 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:15,536 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing ece1749e1d126e57e47de401d52563ef, disabling compactions & flushes 2024-11-18T22:47:15,536 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,536 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,536 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. after waiting 0 ms 2024-11-18T22:47:15,536 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,536 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for ece1749e1d126e57e47de401d52563ef: Waiting for close lock at 1731970035536Disabling compacts and flushes for region at 1731970035536Disabling writes for close at 1731970035536Writing region close event to WAL at 1731970035536Closed at 1731970035536 2024-11-18T22:47:15,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:15,540 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731970035539"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970035539"}]},"ts":"1731970035539"} 2024-11-18T22:47:15,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:15,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:15,547 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970035547"}]},"ts":"1731970035547"} 2024-11-18T22:47:15,554 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-18T22:47:15,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=ece1749e1d126e57e47de401d52563ef, ASSIGN}] 2024-11-18T22:47:15,562 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=ece1749e1d126e57e47de401d52563ef, ASSIGN 2024-11-18T22:47:15,565 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=ece1749e1d126e57e47de401d52563ef, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:15,719 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=ece1749e1d126e57e47de401d52563ef, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:15,721 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=ece1749e1d126e57e47de401d52563ef, ASSIGN because future has completed 2024-11-18T22:47:15,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure ece1749e1d126e57e47de401d52563ef, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:15,880 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,880 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => ece1749e1d126e57e47de401d52563ef, NAME => 'testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:15,881 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,881 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:15,881 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,881 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,884 INFO [StoreOpener-ece1749e1d126e57e47de401d52563ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,886 INFO [StoreOpener-ece1749e1d126e57e47de401d52563ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ece1749e1d126e57e47de401d52563ef columnFamilyName testFamily 2024-11-18T22:47:15,886 DEBUG [StoreOpener-ece1749e1d126e57e47de401d52563ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:15,887 INFO [StoreOpener-ece1749e1d126e57e47de401d52563ef-1 {}] regionserver.HStore(327): Store=ece1749e1d126e57e47de401d52563ef/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:15,887 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,888 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,889 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,890 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,890 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,892 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,896 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:15,897 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened ece1749e1d126e57e47de401d52563ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62749246, jitterRate=-0.06496337056159973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:15,897 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:15,898 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for ece1749e1d126e57e47de401d52563ef: Running coprocessor pre-open hook at 1731970035882Writing region info on filesystem at 1731970035882Initializing all the Stores at 1731970035883 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970035883Cleaning up temporary data from old regions at 1731970035890 (+7 ms)Running coprocessor post-open hooks at 1731970035897 (+7 ms)Region opened successfully at 1731970035898 (+1 ms) 2024-11-18T22:47:15,900 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef., pid=24, masterSystemTime=1731970035874 2024-11-18T22:47:15,907 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,907 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:15,909 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=ece1749e1d126e57e47de401d52563ef, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:15,917 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure ece1749e1d126e57e47de401d52563ef, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:15,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T22:47:15,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure ece1749e1d126e57e47de401d52563ef, server=3b0ca91ed64a,42807,1731969971356 in 198 msec 2024-11-18T22:47:15,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-18T22:47:15,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=ece1749e1d126e57e47de401d52563ef, ASSIGN in 377 msec 2024-11-18T22:47:15,942 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:15,942 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970035942"}]},"ts":"1731970035942"} 2024-11-18T22:47:15,946 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-18T22:47:15,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:15,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 903 msec 2024-11-18T22:47:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-18T22:47:16,073 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-18T22:47:16,073 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-18T22:47:16,073 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:16,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-18T22:47:16,081 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:16,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-18T22:47:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:16,104 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:16,104 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-18T22:47:16,116 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T22:47:16,122 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T22:47:16,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T22:47:16,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-18T22:47:17,278 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:19,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558100' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T22:47:19,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51866 deadline: 1731970040128, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558100' does not exist 2024-11-18T22:47:19,132 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3b0ca91ed64a,42807,1731969971356 for closing scanner id = -581467765047558100 for ece1749e1d126e57e47de401d52563ef of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558100' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T22:47:19,145 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=803 (was 742) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8204 (was 6969) - AvailableMemoryMB LEAK? - 2024-11-18T22:47:19,157 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=803, ProcessCount=11, AvailableMemoryMB=8201 2024-11-18T22:47:19,161 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:19,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-18T22:47:19,173 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:19,173 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:19,175 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-18T22:47:19,176 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T22:47:19,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741849_1025 (size=419) 2024-11-18T22:47:19,613 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9fa27e915f8d2cb580d28129bc214c6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-18T22:47:19,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741850_1026 (size=70) 2024-11-18T22:47:20,048 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:20,049 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing d9fa27e915f8d2cb580d28129bc214c6, disabling compactions & flushes 2024-11-18T22:47:20,049 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,049 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,049 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. after waiting 0 ms 2024-11-18T22:47:20,049 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,049 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,049 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9fa27e915f8d2cb580d28129bc214c6: Waiting for close lock at 1731970040049Disabling compacts and flushes for region at 1731970040049Disabling writes for close at 1731970040049Writing region close event to WAL at 1731970040049Closed at 1731970040049 2024-11-18T22:47:20,057 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:20,057 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731970040057"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970040057"}]},"ts":"1731970040057"} 2024-11-18T22:47:20,064 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:20,068 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:20,068 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970040068"}]},"ts":"1731970040068"} 2024-11-18T22:47:20,074 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-18T22:47:20,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9fa27e915f8d2cb580d28129bc214c6, ASSIGN}] 2024-11-18T22:47:20,077 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9fa27e915f8d2cb580d28129bc214c6, ASSIGN 2024-11-18T22:47:20,079 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9fa27e915f8d2cb580d28129bc214c6, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T22:47:20,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d9fa27e915f8d2cb580d28129bc214c6, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:20,241 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9fa27e915f8d2cb580d28129bc214c6, ASSIGN because future has completed 2024-11-18T22:47:20,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9fa27e915f8d2cb580d28129bc214c6, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:20,443 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => d9fa27e915f8d2cb580d28129bc214c6, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:20,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:20,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,445 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,464 INFO [StoreOpener-d9fa27e915f8d2cb580d28129bc214c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,477 INFO [StoreOpener-d9fa27e915f8d2cb580d28129bc214c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9fa27e915f8d2cb580d28129bc214c6 columnFamilyName testFamily 2024-11-18T22:47:20,477 DEBUG [StoreOpener-d9fa27e915f8d2cb580d28129bc214c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:20,481 INFO [StoreOpener-d9fa27e915f8d2cb580d28129bc214c6-1 {}] regionserver.HStore(327): Store=d9fa27e915f8d2cb580d28129bc214c6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:20,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,488 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,503 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:20,536 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened d9fa27e915f8d2cb580d28129bc214c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70703331, jitterRate=0.053561732172966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:20,536 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:20,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for d9fa27e915f8d2cb580d28129bc214c6: Running coprocessor pre-open hook at 1731970040445Writing region info on filesystem at 1731970040445Initializing all the Stores at 1731970040461 (+16 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970040461Cleaning up temporary data from old regions at 1731970040493 (+32 ms)Running coprocessor post-open hooks at 1731970040536 (+43 ms)Region opened successfully at 1731970040537 (+1 ms) 2024-11-18T22:47:20,542 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., pid=27, masterSystemTime=1731970040420 2024-11-18T22:47:20,553 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,553 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:20,558 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d9fa27e915f8d2cb580d28129bc214c6, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:20,571 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9fa27e915f8d2cb580d28129bc214c6, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:20,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-18T22:47:20,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure d9fa27e915f8d2cb580d28129bc214c6, server=3b0ca91ed64a,42807,1731969971356 in 335 msec 2024-11-18T22:47:20,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T22:47:20,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d9fa27e915f8d2cb580d28129bc214c6, ASSIGN in 527 msec 2024-11-18T22:47:20,626 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:20,627 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970040627"}]},"ts":"1731970040627"} 2024-11-18T22:47:20,645 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-18T22:47:20,647 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:20,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.4860 sec 2024-11-18T22:47:20,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-18T22:47:20,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:20,738 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-18T22:47:20,738 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:20,741 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-18T22:47:20,741 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:21,088 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-18T22:47:21,089 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-18T22:47:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T22:47:22,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-18T22:47:22,220 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-18T22:47:22,221 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:22,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-18T22:47:22,231 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:22,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-18T22:47:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:22,277 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:22,278 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-18T22:47:22,283 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T22:47:22,285 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-18T22:47:25,951 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:25,974 INFO [master/3b0ca91ed64a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T22:47:25,974 INFO [master/3b0ca91ed64a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T22:47:28,287 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-18T22:47:28,287 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-18T22:47:28,287 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:47:28,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=221 (was 222), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=915 (was 803) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6755 (was 8201) 2024-11-18T22:47:28,303 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=915, ProcessCount=11, AvailableMemoryMB=6754 2024-11-18T22:47:28,306 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:28,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-18T22:47:28,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:28,310 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:28,311 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-18T22:47:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T22:47:28,312 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:28,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741851_1027 (size=407) 2024-11-18T22:47:28,731 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 498e9ad760b64ac962cecd6dc7a9d5c8, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:28,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741852_1028 (size=58) 2024-11-18T22:47:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 498e9ad760b64ac962cecd6dc7a9d5c8, disabling compactions & flushes 2024-11-18T22:47:29,188 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. after waiting 0 ms 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,188 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,188 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 498e9ad760b64ac962cecd6dc7a9d5c8: Waiting for close lock at 1731970049188Disabling compacts and flushes for region at 1731970049188Disabling writes for close at 1731970049188Writing region close event to WAL at 1731970049188Closed at 1731970049188 2024-11-18T22:47:29,196 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:29,197 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731970049196"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970049196"}]},"ts":"1731970049196"} 2024-11-18T22:47:29,201 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:29,203 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:29,204 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970049203"}]},"ts":"1731970049203"} 2024-11-18T22:47:29,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-18T22:47:29,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=498e9ad760b64ac962cecd6dc7a9d5c8, ASSIGN}] 2024-11-18T22:47:29,212 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=498e9ad760b64ac962cecd6dc7a9d5c8, ASSIGN 2024-11-18T22:47:29,215 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=498e9ad760b64ac962cecd6dc7a9d5c8, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T22:47:29,367 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=498e9ad760b64ac962cecd6dc7a9d5c8, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:29,374 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=498e9ad760b64ac962cecd6dc7a9d5c8, ASSIGN because future has completed 2024-11-18T22:47:29,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 498e9ad760b64ac962cecd6dc7a9d5c8, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:29,556 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 498e9ad760b64ac962cecd6dc7a9d5c8, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:29,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:29,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,566 INFO [StoreOpener-498e9ad760b64ac962cecd6dc7a9d5c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,581 INFO [StoreOpener-498e9ad760b64ac962cecd6dc7a9d5c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 498e9ad760b64ac962cecd6dc7a9d5c8 columnFamilyName testFamily 2024-11-18T22:47:29,582 DEBUG [StoreOpener-498e9ad760b64ac962cecd6dc7a9d5c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:29,593 INFO [StoreOpener-498e9ad760b64ac962cecd6dc7a9d5c8-1 {}] regionserver.HStore(327): Store=498e9ad760b64ac962cecd6dc7a9d5c8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:29,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,595 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,600 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,624 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:29,632 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 498e9ad760b64ac962cecd6dc7a9d5c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70282700, jitterRate=0.04729384183883667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:29,632 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:29,633 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 498e9ad760b64ac962cecd6dc7a9d5c8: Running coprocessor pre-open hook at 1731970049557Writing region info on filesystem at 1731970049557Initializing all the Stores at 1731970049563 (+6 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970049563Cleaning up temporary data from old regions at 1731970049597 (+34 ms)Running coprocessor post-open hooks at 1731970049632 (+35 ms)Region opened successfully at 1731970049632 2024-11-18T22:47:29,639 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8., pid=30, masterSystemTime=1731970049539 2024-11-18T22:47:29,646 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,647 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:29,650 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=498e9ad760b64ac962cecd6dc7a9d5c8, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:29,659 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 498e9ad760b64ac962cecd6dc7a9d5c8, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:29,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T22:47:29,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 498e9ad760b64ac962cecd6dc7a9d5c8, server=3b0ca91ed64a,42807,1731969971356 in 285 msec 2024-11-18T22:47:29,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-18T22:47:29,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=498e9ad760b64ac962cecd6dc7a9d5c8, ASSIGN in 465 msec 2024-11-18T22:47:29,685 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:29,685 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970049685"}]},"ts":"1731970049685"} 2024-11-18T22:47:29,689 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-18T22:47:29,692 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:29,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.3900 sec 2024-11-18T22:47:30,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-18T22:47:30,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:31,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T22:47:31,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-18T22:47:31,343 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-18T22:47:31,343 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:31,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-18T22:47:31,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:31,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-18T22:47:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:31,410 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:31,410 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-18T22:47:31,415 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T22:47:31,422 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T22:47:31,425 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T22:47:31,427 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-18T22:47:33,109 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-18T22:47:35,060 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:37,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T22:47:37,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42807 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51866 deadline: 1731970063432, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558092' does not exist 2024-11-18T22:47:37,435 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3b0ca91ed64a,42807,1731969971356 for closing scanner id = -581467765047558092 for 498e9ad760b64ac962cecd6dc7a9d5c8 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-581467765047558092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T22:47:37,454 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221 (was 221), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1018 (was 915) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6033 (was 6754) 2024-11-18T22:47:37,485 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1018, ProcessCount=11, AvailableMemoryMB=6031 2024-11-18T22:47:37,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T22:47:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-18T22:47:37,501 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T22:47:37,501 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:37,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-18T22:47:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T22:47:37,513 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T22:47:37,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741853_1029 (size=423) 2024-11-18T22:47:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-18T22:47:37,983 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d5fdf7720596b31e29685e6ad886389, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270 2024-11-18T22:47:38,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741854_1030 (size=74) 2024-11-18T22:47:38,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:38,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0d5fdf7720596b31e29685e6ad886389, disabling compactions & flushes 2024-11-18T22:47:38,045 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. after waiting 0 ms 2024-11-18T22:47:38,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,046 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,046 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d5fdf7720596b31e29685e6ad886389: Waiting for close lock at 1731970058045Disabling compacts and flushes for region at 1731970058045Disabling writes for close at 1731970058045Writing region close event to WAL at 1731970058046 (+1 ms)Closed at 1731970058046 2024-11-18T22:47:38,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T22:47:38,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731970058050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731970058050"}]},"ts":"1731970058050"} 2024-11-18T22:47:38,054 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T22:47:38,056 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T22:47:38,056 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970058056"}]},"ts":"1731970058056"} 2024-11-18T22:47:38,060 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-18T22:47:38,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0d5fdf7720596b31e29685e6ad886389, ASSIGN}] 2024-11-18T22:47:38,063 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0d5fdf7720596b31e29685e6ad886389, ASSIGN 2024-11-18T22:47:38,065 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0d5fdf7720596b31e29685e6ad886389, ASSIGN; state=OFFLINE, location=3b0ca91ed64a,42807,1731969971356; forceNewPlan=false, retain=false 2024-11-18T22:47:38,216 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0d5fdf7720596b31e29685e6ad886389, regionState=OPENING, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:38,225 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0d5fdf7720596b31e29685e6ad886389, ASSIGN because future has completed 2024-11-18T22:47:38,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d5fdf7720596b31e29685e6ad886389, server=3b0ca91ed64a,42807,1731969971356}] 2024-11-18T22:47:38,403 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,403 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d5fdf7720596b31e29685e6ad886389, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389.', STARTKEY => '', ENDKEY => ''} 2024-11-18T22:47:38,403 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,404 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T22:47:38,404 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,404 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,420 INFO [StoreOpener-0d5fdf7720596b31e29685e6ad886389-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,432 INFO [StoreOpener-0d5fdf7720596b31e29685e6ad886389-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d5fdf7720596b31e29685e6ad886389 columnFamilyName testFamily 2024-11-18T22:47:38,432 DEBUG [StoreOpener-0d5fdf7720596b31e29685e6ad886389-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T22:47:38,434 INFO [StoreOpener-0d5fdf7720596b31e29685e6ad886389-1 {}] regionserver.HStore(327): Store=0d5fdf7720596b31e29685e6ad886389/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T22:47:38,434 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,437 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,438 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,441 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,441 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,456 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,469 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T22:47:38,472 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 0d5fdf7720596b31e29685e6ad886389; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65865082, jitterRate=-0.018533796072006226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T22:47:38,473 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:38,473 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 0d5fdf7720596b31e29685e6ad886389: Running coprocessor pre-open hook at 1731970058404Writing region info on filesystem at 1731970058404Initializing all the Stores at 1731970058414 (+10 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731970058414Cleaning up temporary data from old regions at 1731970058441 (+27 ms)Running coprocessor post-open hooks at 1731970058473 (+32 ms)Region opened successfully at 1731970058473 2024-11-18T22:47:38,481 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., pid=33, masterSystemTime=1731970058389 2024-11-18T22:47:38,497 DEBUG [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,497 INFO [RS_OPEN_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:38,505 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=0d5fdf7720596b31e29685e6ad886389, regionState=OPEN, openSeqNum=2, regionLocation=3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:38,518 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d5fdf7720596b31e29685e6ad886389, server=3b0ca91ed64a,42807,1731969971356 because future has completed 2024-11-18T22:47:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T22:47:38,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T22:47:38,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d5fdf7720596b31e29685e6ad886389, server=3b0ca91ed64a,42807,1731969971356 in 295 msec 2024-11-18T22:47:38,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T22:47:38,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=0d5fdf7720596b31e29685e6ad886389, ASSIGN in 464 msec 2024-11-18T22:47:38,532 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T22:47:38,532 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731970058532"}]},"ts":"1731970058532"} 2024-11-18T22:47:38,536 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-18T22:47:38,538 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T22:47:38,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.0450 sec 2024-11-18T22:47:39,112 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-18T22:47:39,188 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T22:47:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T22:47:40,543 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-18T22:47:40,543 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-18T22:47:40,544 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:40,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-18T22:47:40,556 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T22:47:40,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-18T22:47:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:40,636 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T22:47:40,636 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-18T22:47:40,648 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T22:47:40,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T22:47:40,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-18T22:47:40,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T22:47:41,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-18T22:47:41,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-18T22:47:41,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:47:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-18T22:47:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2] 2024-11-18T22:47:42,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42807 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T22:47:43,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3b0ca91ed64a:42807 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=970ms, rpcTimeout=961ms 2024-11-18T22:47:43,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=970ms, rpcTimeout=961ms 2024-11-18T22:47:43,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., hostname=3b0ca91ed64a,42807,1731969971356, seqNum=2 from cache 2024-11-18T22:47:43,677 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=220 (was 221), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1073 (was 1018) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5530 (was 6031) 2024-11-18T22:47:43,678 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T22:47:43,678 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T22:47:43,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T22:47:43,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T22:47:43,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T22:47:43,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T22:47:43,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T22:47:43,693 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T22:47:43,694 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2050315869, stopped=false 2024-11-18T22:47:43,694 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3b0ca91ed64a,36189,1731969970268 2024-11-18T22:47:43,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T22:47:43,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:47:43,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T22:47:43,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T22:47:43,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:47:43,713 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T22:47:43,713 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T22:47:43,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,713 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3b0ca91ed64a,42807,1731969971356' ***** 2024-11-18T22:47:43,714 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T22:47:43,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T22:47:43,714 INFO [regionserver/3b0ca91ed64a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T22:47:43,714 INFO [regionserver/3b0ca91ed64a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T22:47:43,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T22:47:43,715 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T22:47:43,716 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T22:47:43,716 INFO [RS:0;3b0ca91ed64a:42807 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T22:47:43,716 INFO [RS:0;3b0ca91ed64a:42807 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T22:47:43,717 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 498e9ad760b64ac962cecd6dc7a9d5c8 2024-11-18T22:47:43,722 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:43,722 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for bd810abb8d573a852f696b7ab2ca83cc 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 657ca8bc4f923a7997087ddc42f1eae1 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 0d5fdf7720596b31e29685e6ad886389 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 4dda8cbd2ec5ba2edae339bcd4398d1a 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 80d83b5a1b5a68e035c174507dbe0593 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for c51f38b82080c8d15458e3951b949b07 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(959): stopping server 3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T22:47:43,723 INFO [RS:0;3b0ca91ed64a:42807 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3b0ca91ed64a:42807. 2024-11-18T22:47:43,723 DEBUG [RS:0;3b0ca91ed64a:42807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T22:47:43,724 DEBUG [RS:0;3b0ca91ed64a:42807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:43,724 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T22:47:43,724 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 498e9ad760b64ac962cecd6dc7a9d5c8, disabling compactions & flushes 2024-11-18T22:47:43,724 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T22:47:43,724 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:43,724 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T22:47:43,725 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:43,725 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T22:47:43,725 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. after waiting 0 ms 2024-11-18T22:47:43,725 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:43,729 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-18T22:47:43,730 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 498e9ad760b64ac962cecd6dc7a9d5c8, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:43,733 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 498e9ad760b64ac962cecd6dc7a9d5c8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:43,737 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T22:47:43,737 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T22:47:43,738 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T22:47:43,738 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T22:47:43,738 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T22:47:43,738 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-18T22:47:43,870 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/.tmp/testFamily/31aff42064d6476cb902db1c4bf69d6f is 61, key is row-0/testFamily:testQualifier/1731970051363/Put/seqid=0 2024-11-18T22:47:43,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741855_1031 (size=5049) 2024-11-18T22:47:43,907 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T22:47:43,930 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 498e9ad760b64ac962cecd6dc7a9d5c8, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:43,935 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/info/ad0e2aed52e94642a11729538daca518 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1./info:regioninfo/1731970006879/Put/seqid=0 2024-11-18T22:47:44,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741856_1032 (size=20564) 2024-11-18T22:47:44,007 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/info/ad0e2aed52e94642a11729538daca518 2024-11-18T22:47:44,130 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 498e9ad760b64ac962cecd6dc7a9d5c8, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:44,180 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/ns/005419f882c849ba87a2d20731467092 is 43, key is default/ns:d/1731970005799/Put/seqid=0 2024-11-18T22:47:44,195 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T22:47:44,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741857_1033 (size=5153) 2024-11-18T22:47:44,303 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/.tmp/testFamily/31aff42064d6476cb902db1c4bf69d6f 2024-11-18T22:47:44,331 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 498e9ad760b64ac962cecd6dc7a9d5c8, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:44,333 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/.tmp/testFamily/31aff42064d6476cb902db1c4bf69d6f as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/testFamily/31aff42064d6476cb902db1c4bf69d6f 2024-11-18T22:47:44,365 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/testFamily/31aff42064d6476cb902db1c4bf69d6f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:44,377 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 498e9ad760b64ac962cecd6dc7a9d5c8 in 646ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:44,418 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNext/498e9ad760b64ac962cecd6dc7a9d5c8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:44,425 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:44,425 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 498e9ad760b64ac962cecd6dc7a9d5c8: Waiting for close lock at 1731970063724Running coprocessor pre-close hooks at 1731970063724Disabling compacts and flushes for region at 1731970063724Disabling writes for close at 1731970063725 (+1 ms)Obtaining lock to block concurrent updates at 1731970063734 (+9 ms)Preparing flush snapshotting stores in 498e9ad760b64ac962cecd6dc7a9d5c8 at 1731970063734Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970063750 (+16 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. at 1731970063756 (+6 ms)Flushing 498e9ad760b64ac962cecd6dc7a9d5c8/testFamily: creating writer at 1731970063758 (+2 ms)Flushing 498e9ad760b64ac962cecd6dc7a9d5c8/testFamily: appending metadata at 1731970063854 (+96 ms)Flushing 498e9ad760b64ac962cecd6dc7a9d5c8/testFamily: closing flushed file at 1731970063864 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30cf8812: reopening flushed file at 1731970064326 (+462 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 498e9ad760b64ac962cecd6dc7a9d5c8 in 646ms, sequenceid=8, compaction requested=false at 1731970064377 (+51 ms)Writing region close event to WAL at 1731970064387 (+10 ms)Running coprocessor post-close hooks at 1731970064423 (+36 ms)Closed at 1731970064425 (+2 ms) 2024-11-18T22:47:44,426 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731970048306.498e9ad760b64ac962cecd6dc7a9d5c8. 2024-11-18T22:47:44,426 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fbab814c6ccb19ba7363c5ee6d75166e, disabling compactions & flushes 2024-11-18T22:47:44,426 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:44,427 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:44,427 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. after waiting 0 ms 2024-11-18T22:47:44,427 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:44,427 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fbab814c6ccb19ba7363c5ee6d75166e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:44,470 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/.tmp/testFamily/c8efc183e1544e8998246e96e5aa036e is 61, key is row-0/testFamily:testQualifier/1731970024751/Put/seqid=0 2024-11-18T22:47:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741858_1034 (size=5049) 2024-11-18T22:47:44,531 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:44,616 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/ns/005419f882c849ba87a2d20731467092 2024-11-18T22:47:44,660 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/table/17cb0db976834066b61f15c2af5812e7 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731970006908/Put/seqid=0 2024-11-18T22:47:44,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741859_1035 (size=6699) 2024-11-18T22:47:44,690 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/table/17cb0db976834066b61f15c2af5812e7 2024-11-18T22:47:44,725 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/info/ad0e2aed52e94642a11729538daca518 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/info/ad0e2aed52e94642a11729538daca518 2024-11-18T22:47:44,732 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-18T22:47:44,732 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 1588230740, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:44,740 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/info/ad0e2aed52e94642a11729538daca518, entries=100, sequenceid=56, filesize=20.1 K 2024-11-18T22:47:44,745 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/ns/005419f882c849ba87a2d20731467092 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/ns/005419f882c849ba87a2d20731467092 2024-11-18T22:47:44,770 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/ns/005419f882c849ba87a2d20731467092, entries=2, sequenceid=56, filesize=5.0 K 2024-11-18T22:47:44,772 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/.tmp/table/17cb0db976834066b61f15c2af5812e7 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/table/17cb0db976834066b61f15c2af5812e7 2024-11-18T22:47:44,788 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/table/17cb0db976834066b61f15c2af5812e7, entries=20, sequenceid=56, filesize=6.5 K 2024-11-18T22:47:44,797 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1058ms, sequenceid=56, compaction requested=false 2024-11-18T22:47:44,860 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-18T22:47:44,863 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T22:47:44,863 INFO [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T22:47:44,863 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731970063737Running coprocessor pre-close hooks at 1731970063737Disabling compacts and flushes for region at 1731970063737Disabling writes for close at 1731970063738 (+1 ms)Obtaining lock to block concurrent updates at 1731970063738Preparing flush snapshotting stores in 1588230740 at 1731970063738Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731970063755 (+17 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731970063764 (+9 ms)Flushing 1588230740/info: creating writer at 1731970063764Flushing 1588230740/info: appending metadata at 1731970063915 (+151 ms)Flushing 1588230740/info: closing flushed file at 1731970063915Flushing 1588230740/ns: creating writer at 1731970064134 (+219 ms)Flushing 1588230740/ns: appending metadata at 1731970064178 (+44 ms)Flushing 1588230740/ns: closing flushed file at 1731970064178Flushing 1588230740/table: creating writer at 1731970064631 (+453 ms)Flushing 1588230740/table: appending metadata at 1731970064659 (+28 ms)Flushing 1588230740/table: closing flushed file at 1731970064660 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b38ad06: reopening flushed file at 1731970064718 (+58 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c6cde: reopening flushed file at 1731970064741 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@738080c0: reopening flushed file at 1731970064770 (+29 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1058ms, sequenceid=56, compaction requested=false at 1731970064797 (+27 ms)Writing region close event to WAL at 1731970064828 (+31 ms)Running coprocessor post-close hooks at 1731970064863 (+35 ms)Closed at 1731970064863 2024-11-18T22:47:44,864 DEBUG [RS_CLOSE_META-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T22:47:44,896 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/.tmp/testFamily/c8efc183e1544e8998246e96e5aa036e 2024-11-18T22:47:44,933 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef, fbab814c6ccb19ba7363c5ee6d75166e 2024-11-18T22:47:44,950 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/.tmp/testFamily/c8efc183e1544e8998246e96e5aa036e as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/testFamily/c8efc183e1544e8998246e96e5aa036e 2024-11-18T22:47:44,980 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/testFamily/c8efc183e1544e8998246e96e5aa036e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:44,988 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fbab814c6ccb19ba7363c5ee6d75166e in 561ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:45,021 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextException/fbab814c6ccb19ba7363c5ee6d75166e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:45,024 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fbab814c6ccb19ba7363c5ee6d75166e: Waiting for close lock at 1731970064426Running coprocessor pre-close hooks at 1731970064426Disabling compacts and flushes for region at 1731970064426Disabling writes for close at 1731970064427 (+1 ms)Obtaining lock to block concurrent updates at 1731970064427Preparing flush snapshotting stores in fbab814c6ccb19ba7363c5ee6d75166e at 1731970064427Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970064427Flushing stores of testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. at 1731970064431 (+4 ms)Flushing fbab814c6ccb19ba7363c5ee6d75166e/testFamily: creating writer at 1731970064431Flushing fbab814c6ccb19ba7363c5ee6d75166e/testFamily: appending metadata at 1731970064470 (+39 ms)Flushing fbab814c6ccb19ba7363c5ee6d75166e/testFamily: closing flushed file at 1731970064470Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae8f858: reopening flushed file at 1731970064941 (+471 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fbab814c6ccb19ba7363c5ee6d75166e in 561ms, sequenceid=8, compaction requested=false at 1731970064988 (+47 ms)Writing region close event to WAL at 1731970064997 (+9 ms)Running coprocessor post-close hooks at 1731970065024 (+27 ms)Closed at 1731970065024 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731970023717.fbab814c6ccb19ba7363c5ee6d75166e. 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd810abb8d573a852f696b7ab2ca83cc, disabling compactions & flushes 2024-11-18T22:47:45,025 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. after waiting 0 ms 2024-11-18T22:47:45,025 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:45,026 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bd810abb8d573a852f696b7ab2ca83cc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:45,059 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/.tmp/testFamily/cfe88d22795744e2becee332de6925b1 is 61, key is row-0/testFamily:testQualifier/1731970031987/Put/seqid=0 2024-11-18T22:47:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741860_1036 (size=5049) 2024-11-18T22:47:45,089 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/.tmp/testFamily/cfe88d22795744e2becee332de6925b1 2024-11-18T22:47:45,115 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/.tmp/testFamily/cfe88d22795744e2becee332de6925b1 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/testFamily/cfe88d22795744e2becee332de6925b1 2024-11-18T22:47:45,133 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 4dda8cbd2ec5ba2edae339bcd4398d1a, 657ca8bc4f923a7997087ddc42f1eae1, 80d83b5a1b5a68e035c174507dbe0593, bd810abb8d573a852f696b7ab2ca83cc, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:45,136 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/testFamily/cfe88d22795744e2becee332de6925b1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:45,138 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd810abb8d573a852f696b7ab2ca83cc in 112ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:45,160 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNextAsync/bd810abb8d573a852f696b7ab2ca83cc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:45,163 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd810abb8d573a852f696b7ab2ca83cc: Waiting for close lock at 1731970065025Running coprocessor pre-close hooks at 1731970065025Disabling compacts and flushes for region at 1731970065025Disabling writes for close at 1731970065025Obtaining lock to block concurrent updates at 1731970065026 (+1 ms)Preparing flush snapshotting stores in bd810abb8d573a852f696b7ab2ca83cc at 1731970065026Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970065026Flushing stores of testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. at 1731970065027 (+1 ms)Flushing bd810abb8d573a852f696b7ab2ca83cc/testFamily: creating writer at 1731970065028 (+1 ms)Flushing bd810abb8d573a852f696b7ab2ca83cc/testFamily: appending metadata at 1731970065058 (+30 ms)Flushing bd810abb8d573a852f696b7ab2ca83cc/testFamily: closing flushed file at 1731970065058Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b3f91dc: reopening flushed file at 1731970065111 (+53 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd810abb8d573a852f696b7ab2ca83cc in 112ms, sequenceid=8, compaction requested=false at 1731970065138 (+27 ms)Writing region close event to WAL at 1731970065149 (+11 ms)Running coprocessor post-close hooks at 1731970065163 (+14 ms)Closed at 1731970065163 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731970030947.bd810abb8d573a852f696b7ab2ca83cc. 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 657ca8bc4f923a7997087ddc42f1eae1, disabling compactions & flushes 2024-11-18T22:47:45,164 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. after waiting 0 ms 2024-11-18T22:47:45,164 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:47:45,164 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 657ca8bc4f923a7997087ddc42f1eae1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:45,198 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/.tmp/testFamily/80fcd15840cb486bb56ec0faeb3768ff is 61, key is row-0/testFamily:testQualifier/1731970007335/Put/seqid=0 2024-11-18T22:47:45,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741861_1037 (size=5049) 2024-11-18T22:47:45,232 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/.tmp/testFamily/80fcd15840cb486bb56ec0faeb3768ff 2024-11-18T22:47:45,278 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/.tmp/testFamily/80fcd15840cb486bb56ec0faeb3768ff as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/testFamily/80fcd15840cb486bb56ec0faeb3768ff 2024-11-18T22:47:45,299 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/testFamily/80fcd15840cb486bb56ec0faeb3768ff, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:45,301 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 657ca8bc4f923a7997087ddc42f1eae1 in 136ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:45,330 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testRetryOutOfOrderScannerNextExceptionAsync/657ca8bc4f923a7997087ddc42f1eae1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:45,333 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:47:45,334 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 657ca8bc4f923a7997087ddc42f1eae1: Waiting for close lock at 1731970065164Running coprocessor pre-close hooks at 1731970065164Disabling compacts and flushes for region at 1731970065164Disabling writes for close at 1731970065164Obtaining lock to block concurrent updates at 1731970065165 (+1 ms)Preparing flush snapshotting stores in 657ca8bc4f923a7997087ddc42f1eae1 at 1731970065165Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970065165Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. at 1731970065168 (+3 ms)Flushing 657ca8bc4f923a7997087ddc42f1eae1/testFamily: creating writer at 1731970065168Flushing 657ca8bc4f923a7997087ddc42f1eae1/testFamily: appending metadata at 1731970065198 (+30 ms)Flushing 657ca8bc4f923a7997087ddc42f1eae1/testFamily: closing flushed file at 1731970065198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63031b5f: reopening flushed file at 1731970065272 (+74 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 657ca8bc4f923a7997087ddc42f1eae1 in 136ms, sequenceid=8, compaction requested=false at 1731970065301 (+29 ms)Writing region close event to WAL at 1731970065315 (+14 ms)Running coprocessor post-close hooks at 1731970065333 (+18 ms)Closed at 1731970065333 2024-11-18T22:47:45,334 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731970006225.657ca8bc4f923a7997087ddc42f1eae1. 2024-11-18T22:47:45,334 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ece1749e1d126e57e47de401d52563ef, disabling compactions & flushes 2024-11-18T22:47:45,334 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:45,335 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:45,335 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6, ece1749e1d126e57e47de401d52563ef 2024-11-18T22:47:45,335 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. after waiting 0 ms 2024-11-18T22:47:45,335 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:45,335 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ece1749e1d126e57e47de401d52563ef 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:45,373 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/.tmp/testFamily/8f8a932a40c64fababb8b1bbe436e29d is 61, key is row-0/testFamily:testQualifier/1731970036088/Put/seqid=0 2024-11-18T22:47:45,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741862_1038 (size=5049) 2024-11-18T22:47:45,406 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/.tmp/testFamily/8f8a932a40c64fababb8b1bbe436e29d 2024-11-18T22:47:45,431 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/.tmp/testFamily/8f8a932a40c64fababb8b1bbe436e29d as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/testFamily/8f8a932a40c64fababb8b1bbe436e29d 2024-11-18T22:47:45,469 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/testFamily/8f8a932a40c64fababb8b1bbe436e29d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:45,473 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ece1749e1d126e57e47de401d52563ef in 138ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:45,505 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnNext/ece1749e1d126e57e47de401d52563ef/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:45,511 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:45,513 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ece1749e1d126e57e47de401d52563ef: Waiting for close lock at 1731970065334Running coprocessor pre-close hooks at 1731970065334Disabling compacts and flushes for region at 1731970065334Disabling writes for close at 1731970065335 (+1 ms)Obtaining lock to block concurrent updates at 1731970065335Preparing flush snapshotting stores in ece1749e1d126e57e47de401d52563ef at 1731970065335Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970065335Flushing stores of testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. at 1731970065338 (+3 ms)Flushing ece1749e1d126e57e47de401d52563ef/testFamily: creating writer at 1731970065339 (+1 ms)Flushing ece1749e1d126e57e47de401d52563ef/testFamily: appending metadata at 1731970065373 (+34 ms)Flushing ece1749e1d126e57e47de401d52563ef/testFamily: closing flushed file at 1731970065373Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22d5a985: reopening flushed file at 1731970065429 (+56 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ece1749e1d126e57e47de401d52563ef in 138ms, sequenceid=8, compaction requested=false at 1731970065473 (+44 ms)Writing region close event to WAL at 1731970065479 (+6 ms)Running coprocessor post-close hooks at 1731970065511 (+32 ms)Closed at 1731970065511 2024-11-18T22:47:45,514 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731970035044.ece1749e1d126e57e47de401d52563ef. 2024-11-18T22:47:45,514 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0d5fdf7720596b31e29685e6ad886389, disabling compactions & flushes 2024-11-18T22:47:45,514 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:45,514 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:45,514 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. after waiting 0 ms 2024-11-18T22:47:45,515 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:45,515 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0d5fdf7720596b31e29685e6ad886389 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:45,536 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 0d5fdf7720596b31e29685e6ad886389, 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:45,560 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/.tmp/testFamily/ec52931649534eeb919dc744b4694ab7 is 61, key is row-0/testFamily:testQualifier/1731970060572/Put/seqid=0 2024-11-18T22:47:45,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741863_1039 (size=5049) 2024-11-18T22:47:45,601 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/.tmp/testFamily/ec52931649534eeb919dc744b4694ab7 2024-11-18T22:47:45,645 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/.tmp/testFamily/ec52931649534eeb919dc744b4694ab7 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/testFamily/ec52931649534eeb919dc744b4694ab7 2024-11-18T22:47:45,664 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/testFamily/ec52931649534eeb919dc744b4694ab7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:45,676 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0d5fdf7720596b31e29685e6ad886389 in 161ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:45,712 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScannerAsync/0d5fdf7720596b31e29685e6ad886389/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:45,714 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:45,715 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0d5fdf7720596b31e29685e6ad886389: Waiting for close lock at 1731970065514Running coprocessor pre-close hooks at 1731970065514Disabling compacts and flushes for region at 1731970065514Disabling writes for close at 1731970065515 (+1 ms)Obtaining lock to block concurrent updates at 1731970065515Preparing flush snapshotting stores in 0d5fdf7720596b31e29685e6ad886389 at 1731970065515Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970065516 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. at 1731970065519 (+3 ms)Flushing 0d5fdf7720596b31e29685e6ad886389/testFamily: creating writer at 1731970065521 (+2 ms)Flushing 0d5fdf7720596b31e29685e6ad886389/testFamily: appending metadata at 1731970065560 (+39 ms)Flushing 0d5fdf7720596b31e29685e6ad886389/testFamily: closing flushed file at 1731970065560Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9d73a80: reopening flushed file at 1731970065641 (+81 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0d5fdf7720596b31e29685e6ad886389 in 161ms, sequenceid=8, compaction requested=false at 1731970065676 (+35 ms)Writing region close event to WAL at 1731970065684 (+8 ms)Running coprocessor post-close hooks at 1731970065714 (+30 ms)Closed at 1731970065714 2024-11-18T22:47:45,715 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731970057493.0d5fdf7720596b31e29685e6ad886389. 2024-11-18T22:47:45,715 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4dda8cbd2ec5ba2edae339bcd4398d1a, disabling compactions & flushes 2024-11-18T22:47:45,715 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:47:45,715 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:47:45,716 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. after waiting 0 ms 2024-11-18T22:47:45,716 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:47:45,716 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4dda8cbd2ec5ba2edae339bcd4398d1a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:45,740 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-18T22:47:45,740 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1325): Online Regions={4dda8cbd2ec5ba2edae339bcd4398d1a=hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a., 80d83b5a1b5a68e035c174507dbe0593=hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., c51f38b82080c8d15458e3951b949b07=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., d9fa27e915f8d2cb580d28129bc214c6=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.} 2024-11-18T22:47:45,740 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:45,751 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/.tmp/testFamily/ba6fe4a2855540b29f3c51ba71a572de is 61, key is row-0/testFamily:testQualifier/1731970017676/Put/seqid=0 2024-11-18T22:47:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741864_1040 (size=5049) 2024-11-18T22:47:45,941 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:46,141 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:46,341 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:46,425 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/.tmp/testFamily/ba6fe4a2855540b29f3c51ba71a572de 2024-11-18T22:47:46,499 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/.tmp/testFamily/ba6fe4a2855540b29f3c51ba71a572de as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily/ba6fe4a2855540b29f3c51ba71a572de 2024-11-18T22:47:46,533 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily/ba6fe4a2855540b29f3c51ba71a572de, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:46,536 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4dda8cbd2ec5ba2edae339bcd4398d1a in 819ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:46,542 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on 4dda8cbd2ec5ba2edae339bcd4398d1a, 80d83b5a1b5a68e035c174507dbe0593, c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:46,545 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnNextAsync/4dda8cbd2ec5ba2edae339bcd4398d1a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:46,547 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:47:46,547 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4dda8cbd2ec5ba2edae339bcd4398d1a: Waiting for close lock at 1731970065715Running coprocessor pre-close hooks at 1731970065715Disabling compacts and flushes for region at 1731970065715Disabling writes for close at 1731970065716 (+1 ms)Obtaining lock to block concurrent updates at 1731970065716Preparing flush snapshotting stores in 4dda8cbd2ec5ba2edae339bcd4398d1a at 1731970065716Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970065716Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. at 1731970065718 (+2 ms)Flushing 4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily: creating writer at 1731970065718Flushing 4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily: appending metadata at 1731970065751 (+33 ms)Flushing 4dda8cbd2ec5ba2edae339bcd4398d1a/testFamily: closing flushed file at 1731970065751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47aa4ec5: reopening flushed file at 1731970066498 (+747 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4dda8cbd2ec5ba2edae339bcd4398d1a in 819ms, sequenceid=8, compaction requested=false at 1731970066536 (+38 ms)Writing region close event to WAL at 1731970066538 (+2 ms)Running coprocessor post-close hooks at 1731970066547 (+9 ms)Closed at 1731970066547 2024-11-18T22:47:46,548 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731970014625.4dda8cbd2ec5ba2edae339bcd4398d1a. 2024-11-18T22:47:46,548 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80d83b5a1b5a68e035c174507dbe0593, disabling compactions & flushes 2024-11-18T22:47:46,548 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:47:46,548 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:47:46,548 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. after waiting 0 ms 2024-11-18T22:47:46,548 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:47:46,548 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 80d83b5a1b5a68e035c174507dbe0593 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:46,571 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/.tmp/testFamily/c76b0c7de1b94ef682e9fcaf1439526f is 61, key is row-0/testFamily:testQualifier/1731970008564/Put/seqid=0 2024-11-18T22:47:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741865_1041 (size=5049) 2024-11-18T22:47:46,678 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/.tmp/testFamily/c76b0c7de1b94ef682e9fcaf1439526f 2024-11-18T22:47:46,694 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/.tmp/testFamily/c76b0c7de1b94ef682e9fcaf1439526f as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/testFamily/c76b0c7de1b94ef682e9fcaf1439526f 2024-11-18T22:47:46,702 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/testFamily/c76b0c7de1b94ef682e9fcaf1439526f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:46,703 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80d83b5a1b5a68e035c174507dbe0593 in 155ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:46,718 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScanner/80d83b5a1b5a68e035c174507dbe0593/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:46,719 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80d83b5a1b5a68e035c174507dbe0593: Waiting for close lock at 1731970066548Running coprocessor pre-close hooks at 1731970066548Disabling compacts and flushes for region at 1731970066548Disabling writes for close at 1731970066548Obtaining lock to block concurrent updates at 1731970066548Preparing flush snapshotting stores in 80d83b5a1b5a68e035c174507dbe0593 at 1731970066548Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970066549 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. at 1731970066549Flushing 80d83b5a1b5a68e035c174507dbe0593/testFamily: creating writer at 1731970066549Flushing 80d83b5a1b5a68e035c174507dbe0593/testFamily: appending metadata at 1731970066571 (+22 ms)Flushing 80d83b5a1b5a68e035c174507dbe0593/testFamily: closing flushed file at 1731970066571Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f592835: reopening flushed file at 1731970066692 (+121 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80d83b5a1b5a68e035c174507dbe0593 in 155ms, sequenceid=8, compaction requested=false at 1731970066704 (+12 ms)Writing region close event to WAL at 1731970066708 (+4 ms)Running coprocessor post-close hooks at 1731970066719 (+11 ms)Closed at 1731970066719 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731970007505.80d83b5a1b5a68e035c174507dbe0593. 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c51f38b82080c8d15458e3951b949b07, disabling compactions & flushes 2024-11-18T22:47:46,720 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. after waiting 0 ms 2024-11-18T22:47:46,720 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:46,721 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c51f38b82080c8d15458e3951b949b07 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:46,742 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T22:47:46,742 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1325): Online Regions={c51f38b82080c8d15458e3951b949b07=testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., d9fa27e915f8d2cb580d28129bc214c6=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6.} 2024-11-18T22:47:46,742 DEBUG [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1351): Waiting on c51f38b82080c8d15458e3951b949b07, d9fa27e915f8d2cb580d28129bc214c6 2024-11-18T22:47:46,746 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/.tmp/testFamily/6a6cec00712a46948a9f0f83f2ffa2e2 is 61, key is row-0/testFamily:testQualifier/1731970027895/Put/seqid=0 2024-11-18T22:47:46,749 DEBUG [master/3b0ca91ed64a:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T22:47:46,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741866_1042 (size=5049) 2024-11-18T22:47:46,758 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/.tmp/testFamily/6a6cec00712a46948a9f0f83f2ffa2e2 2024-11-18T22:47:46,771 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/.tmp/testFamily/6a6cec00712a46948a9f0f83f2ffa2e2 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/testFamily/6a6cec00712a46948a9f0f83f2ffa2e2 2024-11-18T22:47:46,793 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/testFamily/6a6cec00712a46948a9f0f83f2ffa2e2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:46,796 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c51f38b82080c8d15458e3951b949b07 in 75ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:46,840 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/default/testNormalScanTimeoutOnOpenScanner/c51f38b82080c8d15458e3951b949b07/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:46,843 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:46,843 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c51f38b82080c8d15458e3951b949b07: Waiting for close lock at 1731970066720Running coprocessor pre-close hooks at 1731970066720Disabling compacts and flushes for region at 1731970066720Disabling writes for close at 1731970066720Obtaining lock to block concurrent updates at 1731970066721 (+1 ms)Preparing flush snapshotting stores in c51f38b82080c8d15458e3951b949b07 at 1731970066721Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970066721Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. at 1731970066722 (+1 ms)Flushing c51f38b82080c8d15458e3951b949b07/testFamily: creating writer at 1731970066722Flushing c51f38b82080c8d15458e3951b949b07/testFamily: appending metadata at 1731970066746 (+24 ms)Flushing c51f38b82080c8d15458e3951b949b07/testFamily: closing flushed file at 1731970066746Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63fa3310: reopening flushed file at 1731970066769 (+23 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c51f38b82080c8d15458e3951b949b07 in 75ms, sequenceid=8, compaction requested=false at 1731970066796 (+27 ms)Writing region close event to WAL at 1731970066828 (+32 ms)Running coprocessor post-close hooks at 1731970066842 (+14 ms)Closed at 1731970066842 2024-11-18T22:47:46,843 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731970024848.c51f38b82080c8d15458e3951b949b07. 2024-11-18T22:47:46,843 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d9fa27e915f8d2cb580d28129bc214c6, disabling compactions & flushes 2024-11-18T22:47:46,844 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:46,844 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:46,844 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. after waiting 0 ms 2024-11-18T22:47:46,844 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:46,844 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d9fa27e915f8d2cb580d28129bc214c6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T22:47:46,867 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/.tmp/testFamily/173b3cb881a74deebcda70206fe45a54 is 61, key is row-0/testFamily:testQualifier/1731970042242/Put/seqid=0 2024-11-18T22:47:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741867_1043 (size=5049) 2024-11-18T22:47:46,881 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/.tmp/testFamily/173b3cb881a74deebcda70206fe45a54 2024-11-18T22:47:46,892 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/.tmp/testFamily/173b3cb881a74deebcda70206fe45a54 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/testFamily/173b3cb881a74deebcda70206fe45a54 2024-11-18T22:47:46,905 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/testFamily/173b3cb881a74deebcda70206fe45a54, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T22:47:46,907 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d9fa27e915f8d2cb580d28129bc214c6 in 63ms, sequenceid=8, compaction requested=false 2024-11-18T22:47:46,915 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d9fa27e915f8d2cb580d28129bc214c6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T22:47:46,916 INFO [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:46,916 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d9fa27e915f8d2cb580d28129bc214c6: Waiting for close lock at 1731970066843Running coprocessor pre-close hooks at 1731970066843Disabling compacts and flushes for region at 1731970066843Disabling writes for close at 1731970066844 (+1 ms)Obtaining lock to block concurrent updates at 1731970066844Preparing flush snapshotting stores in d9fa27e915f8d2cb580d28129bc214c6 at 1731970066844Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731970066844Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. at 1731970066846 (+2 ms)Flushing d9fa27e915f8d2cb580d28129bc214c6/testFamily: creating writer at 1731970066846Flushing d9fa27e915f8d2cb580d28129bc214c6/testFamily: appending metadata at 1731970066867 (+21 ms)Flushing d9fa27e915f8d2cb580d28129bc214c6/testFamily: closing flushed file at 1731970066867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52122fa4: reopening flushed file at 1731970066890 (+23 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d9fa27e915f8d2cb580d28129bc214c6 in 63ms, sequenceid=8, compaction requested=false at 1731970066907 (+17 ms)Writing region close event to WAL at 1731970066908 (+1 ms)Running coprocessor post-close hooks at 1731970066916 (+8 ms)Closed at 1731970066916 2024-11-18T22:47:46,916 DEBUG [RS_CLOSE_REGION-regionserver/3b0ca91ed64a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731970039160.d9fa27e915f8d2cb580d28129bc214c6. 2024-11-18T22:47:46,943 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(976): stopping server 3b0ca91ed64a,42807,1731969971356; all regions closed. 2024-11-18T22:47:46,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741834_1010 (size=19479) 2024-11-18T22:47:46,959 DEBUG [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/oldWALs 2024-11-18T22:47:46,959 INFO [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b0ca91ed64a%2C42807%2C1731969971356.meta:.meta(num 1731970005406) 2024-11-18T22:47:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741833_1009 (size=20031) 2024-11-18T22:47:46,964 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/WALs/3b0ca91ed64a,42807,1731969971356/3b0ca91ed64a%2C42807%2C1731969971356.1731970004857 not finished, retry = 0 2024-11-18T22:47:47,068 DEBUG [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/oldWALs 2024-11-18T22:47:47,068 INFO [RS:0;3b0ca91ed64a:42807 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b0ca91ed64a%2C42807%2C1731969971356:(num 1731970004857) 2024-11-18T22:47:47,068 DEBUG [RS:0;3b0ca91ed64a:42807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T22:47:47,068 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T22:47:47,068 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T22:47:47,068 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.ChoreService(370): Chore service for: regionserver/3b0ca91ed64a:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T22:47:47,069 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T22:47:47,069 INFO [regionserver/3b0ca91ed64a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T22:47:47,070 INFO [RS:0;3b0ca91ed64a:42807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42807 2024-11-18T22:47:47,080 INFO [regionserver/3b0ca91ed64a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T22:47:47,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3b0ca91ed64a,42807,1731969971356 2024-11-18T22:47:47,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T22:47:47,103 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T22:47:47,124 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3b0ca91ed64a,42807,1731969971356] 2024-11-18T22:47:47,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3b0ca91ed64a,42807,1731969971356 already deleted, retry=false 2024-11-18T22:47:47,145 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3b0ca91ed64a,42807,1731969971356 expired; onlineServers=0 2024-11-18T22:47:47,145 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3b0ca91ed64a,36189,1731969970268' ***** 2024-11-18T22:47:47,145 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T22:47:47,145 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T22:47:47,145 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T22:47:47,146 DEBUG [M:0;3b0ca91ed64a:36189 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T22:47:47,146 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T22:47:47,146 DEBUG [M:0;3b0ca91ed64a:36189 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T22:47:47,146 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.small.0-1731969974466 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.small.0-1731969974466,5,FailOnTimeoutGroup] 2024-11-18T22:47:47,146 DEBUG [master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.large.0-1731969974458 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b0ca91ed64a:0:becomeActiveMaster-HFileCleaner.large.0-1731969974458,5,FailOnTimeoutGroup] 2024-11-18T22:47:47,146 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.ChoreService(370): Chore service for: master/3b0ca91ed64a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T22:47:47,146 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T22:47:47,146 DEBUG [M:0;3b0ca91ed64a:36189 {}] master.HMaster(1795): Stopping service threads 2024-11-18T22:47:47,146 INFO [M:0;3b0ca91ed64a:36189 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T22:47:47,147 INFO [M:0;3b0ca91ed64a:36189 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T22:47:47,147 INFO [M:0;3b0ca91ed64a:36189 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T22:47:47,147 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T22:47:47,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T22:47:47,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T22:47:47,187 DEBUG [M:0;3b0ca91ed64a:36189 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-18T22:47:47,187 DEBUG [M:0;3b0ca91ed64a:36189 {}] master.ActiveMasterManager(353): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-18T22:47:47,188 INFO [M:0;3b0ca91ed64a:36189 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/.lastflushedseqids 2024-11-18T22:47:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741868_1044 (size=460) 2024-11-18T22:47:47,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T22:47:47,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42807-0x1015020684e0001, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T22:47:47,225 INFO [RS:0;3b0ca91ed64a:42807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T22:47:47,225 INFO [RS:0;3b0ca91ed64a:42807 {}] regionserver.HRegionServer(1031): Exiting; stopping=3b0ca91ed64a,42807,1731969971356; zookeeper connection closed. 2024-11-18T22:47:47,226 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3530c865 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3530c865 2024-11-18T22:47:47,227 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T22:47:47,608 INFO [M:0;3b0ca91ed64a:36189 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T22:47:47,608 INFO [M:0;3b0ca91ed64a:36189 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T22:47:47,609 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T22:47:47,609 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:47:47,609 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:47:47,609 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T22:47:47,609 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:47:47,609 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.89 KB heapSize=190.38 KB 2024-11-18T22:47:47,640 DEBUG [M:0;3b0ca91ed64a:36189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11e1ab9b63ac457eb0668abbaebe8ed9 is 82, key is hbase:meta,,1/info:regioninfo/1731970005605/Put/seqid=0 2024-11-18T22:47:47,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741869_1045 (size=5672) 2024-11-18T22:47:47,654 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11e1ab9b63ac457eb0668abbaebe8ed9 2024-11-18T22:47:47,685 DEBUG [M:0;3b0ca91ed64a:36189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1a4d9afd37640e9956f090226190089 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731970006919/Put/seqid=0 2024-11-18T22:47:47,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741870_1046 (size=14077) 2024-11-18T22:47:48,103 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.33 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1a4d9afd37640e9956f090226190089 2024-11-18T22:47:48,109 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1a4d9afd37640e9956f090226190089 2024-11-18T22:47:48,130 DEBUG [M:0;3b0ca91ed64a:36189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba0cfd734e774dac92cb089e59b9250e is 69, key is 3b0ca91ed64a,42807,1731969971356/rs:state/1731969974558/Put/seqid=0 2024-11-18T22:47:48,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741871_1047 (size=5156) 2024-11-18T22:47:48,536 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba0cfd734e774dac92cb089e59b9250e 2024-11-18T22:47:48,547 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/11e1ab9b63ac457eb0668abbaebe8ed9 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11e1ab9b63ac457eb0668abbaebe8ed9 2024-11-18T22:47:48,559 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/11e1ab9b63ac457eb0668abbaebe8ed9, entries=8, sequenceid=311, filesize=5.5 K 2024-11-18T22:47:48,561 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1a4d9afd37640e9956f090226190089 as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1a4d9afd37640e9956f090226190089 2024-11-18T22:47:48,575 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1a4d9afd37640e9956f090226190089 2024-11-18T22:47:48,575 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1a4d9afd37640e9956f090226190089, entries=33, sequenceid=311, filesize=13.7 K 2024-11-18T22:47:48,577 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ba0cfd734e774dac92cb089e59b9250e as hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ba0cfd734e774dac92cb089e59b9250e 2024-11-18T22:47:48,586 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37885/user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ba0cfd734e774dac92cb089e59b9250e, entries=1, sequenceid=311, filesize=5.0 K 2024-11-18T22:47:48,587 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.89 KB/166795, heapSize ~190.08 KB/194640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 978ms, sequenceid=311, compaction requested=false 2024-11-18T22:47:48,594 INFO [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T22:47:48,595 DEBUG [M:0;3b0ca91ed64a:36189 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731970067608Disabling compacts and flushes for region at 1731970067608Disabling writes for close at 1731970067609 (+1 ms)Obtaining lock to block concurrent updates at 1731970067609Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731970067609Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166795, getHeapSize=194880, getOffHeapSize=0, getCellsCount=358 at 1731970067610 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731970067611 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731970067611Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731970067640 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731970067640Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731970067664 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731970067684 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731970067684Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731970068110 (+426 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731970068129 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731970068129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46606db7: reopening flushed file at 1731970068545 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@567efd21: reopening flushed file at 1731970068559 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@708bef01: reopening flushed file at 1731970068576 (+17 ms)Finished flush of dataSize ~162.89 KB/166795, heapSize ~190.08 KB/194640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 978ms, sequenceid=311, compaction requested=false at 1731970068587 (+11 ms)Writing region close event to WAL at 1731970068594 (+7 ms)Closed at 1731970068594 2024-11-18T22:47:48,611 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/47c6aff4-e210-6155-8a69-37920ef8d270/MasterData/WALs/3b0ca91ed64a,36189,1731969970268/3b0ca91ed64a%2C36189%2C1731969970268.1731969973659 not finished, retry = 0 2024-11-18T22:47:48,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43817 is added to blk_1073741830_1006 (size=187617) 2024-11-18T22:47:48,713 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T22:47:48,713 INFO [M:0;3b0ca91ed64a:36189 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T22:47:48,713 INFO [M:0;3b0ca91ed64a:36189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36189 2024-11-18T22:47:48,713 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T22:47:48,824 INFO [M:0;3b0ca91ed64a:36189 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T22:47:48,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T22:47:48,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36189-0x1015020684e0000, quorum=127.0.0.1:60767, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T22:47:48,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f9a1a82{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T22:47:48,837 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@210254cd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T22:47:48,837 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T22:47:48,837 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23c598c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T22:47:48,837 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7aa82d16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.log.dir/,STOPPED} 2024-11-18T22:47:48,845 WARN [BP-1011012363-172.17.0.2-1731969965710 heartbeating to localhost/127.0.0.1:37885 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T22:47:48,845 WARN [BP-1011012363-172.17.0.2-1731969965710 heartbeating to localhost/127.0.0.1:37885 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1011012363-172.17.0.2-1731969965710 (Datanode Uuid ec45131f-66ab-4b5b-b786-7aeebb4afcc0) service to localhost/127.0.0.1:37885 2024-11-18T22:47:48,846 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T22:47:48,846 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T22:47:48,847 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/data/data1/current/BP-1011012363-172.17.0.2-1731969965710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T22:47:48,847 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/cluster_a571b389-7fae-1a26-8e04-1bf407ea45aa/data/data2/current/BP-1011012363-172.17.0.2-1731969965710 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T22:47:48,847 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T22:47:48,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4df87c21{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T22:47:48,869 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T22:47:48,869 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T22:47:48,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T22:47:48,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/596846df-088f-cdaf-080f-d558cb890d70/hadoop.log.dir/,STOPPED} 2024-11-18T22:47:48,882 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T22:47:48,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down