2024-11-20 23:40:05,103 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 23:40:05,121 main DEBUG Took 0.016261 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 23:40:05,121 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 23:40:05,122 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 23:40:05,123 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 23:40:05,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,133 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 23:40:05,149 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,153 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,156 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,157 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,159 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,160 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,161 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,162 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,163 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,163 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,164 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,166 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 23:40:05,167 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,167 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 23:40:05,169 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 23:40:05,171 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 23:40:05,173 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 23:40:05,174 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 23:40:05,175 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 23:40:05,176 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 23:40:05,186 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 23:40:05,190 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 23:40:05,192 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 23:40:05,192 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 23:40:05,193 main DEBUG createAppenders(={Console}) 2024-11-20 23:40:05,194 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-20 23:40:05,194 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 23:40:05,195 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-20 23:40:05,195 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 23:40:05,195 main DEBUG OutputStream closed 2024-11-20 23:40:05,196 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 23:40:05,196 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 23:40:05,196 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-20 23:40:05,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 23:40:05,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 23:40:05,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 23:40:05,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 23:40:05,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 23:40:05,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 23:40:05,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 23:40:05,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 23:40:05,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 23:40:05,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 23:40:05,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 23:40:05,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 23:40:05,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 23:40:05,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 23:40:05,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 23:40:05,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 23:40:05,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 23:40:05,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 23:40:05,286 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 23:40:05,286 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-20 23:40:05,286 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 23:40:05,287 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-20T23:40:05,304 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-20 23:40:05,307 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 23:40:05,307 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T23:40:05,550 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5 2024-11-20T23:40:05,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T23:40:05,585 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7, deleteOnExit=true 2024-11-20T23:40:05,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T23:40:05,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/test.cache.data in system properties and HBase conf 2024-11-20T23:40:05,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T23:40:05,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.log.dir in system properties and HBase conf 2024-11-20T23:40:05,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T23:40:05,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T23:40:05,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T23:40:05,674 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T23:40:05,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T23:40:05,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T23:40:05,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T23:40:05,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T23:40:05,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T23:40:05,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T23:40:05,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T23:40:05,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T23:40:05,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T23:40:05,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T23:40:05,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/nfs.dump.dir in system properties and HBase conf 2024-11-20T23:40:05,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/java.io.tmpdir in system properties and HBase conf 2024-11-20T23:40:05,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T23:40:05,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T23:40:05,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T23:40:06,934 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T23:40:07,008 INFO [Time-limited test {}] log.Log(170): Logging initialized @2600ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T23:40:07,072 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T23:40:07,129 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T23:40:07,150 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T23:40:07,151 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T23:40:07,152 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T23:40:07,165 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T23:40:07,168 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.log.dir/,AVAILABLE} 2024-11-20T23:40:07,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T23:40:07,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/java.io.tmpdir/jetty-localhost-42809-hadoop-hdfs-3_4_1-tests_jar-_-any-9017051929356809806/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T23:40:07,362 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:42809} 2024-11-20T23:40:07,362 INFO [Time-limited test {}] server.Server(415): Started @2955ms 2024-11-20T23:40:08,008 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T23:40:08,015 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T23:40:08,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T23:40:08,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T23:40:08,017 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T23:40:08,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.log.dir/,AVAILABLE} 2024-11-20T23:40:08,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T23:40:08,116 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc78d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/java.io.tmpdir/jetty-localhost-43219-hadoop-hdfs-3_4_1-tests_jar-_-any-12370579948727883456/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T23:40:08,117 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:43219} 2024-11-20T23:40:08,117 INFO [Time-limited test {}] server.Server(415): Started @3711ms 2024-11-20T23:40:08,164 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T23:40:09,193 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/data/data2/current/BP-2020952472-172.17.0.2-1732146006291/current, will proceed with Du for space computation calculation, 2024-11-20T23:40:09,193 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/data/data1/current/BP-2020952472-172.17.0.2-1732146006291/current, will proceed with Du for space computation calculation, 2024-11-20T23:40:09,222 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T23:40:09,267 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5b7a408380e75e22 with lease ID 0x70efc4c9f27e94d0: Processing first storage report for DS-97311b13-b495-468d-ad3e-c04052ad0732 from datanode DatanodeRegistration(127.0.0.1:35447, datanodeUuid=942ccc57-68d5-4f40-972f-449228f06d9f, infoPort=37781, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=2098193195;c=1732146006291) 2024-11-20T23:40:09,268 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5b7a408380e75e22 with lease ID 0x70efc4c9f27e94d0: from storage DS-97311b13-b495-468d-ad3e-c04052ad0732 node DatanodeRegistration(127.0.0.1:35447, datanodeUuid=942ccc57-68d5-4f40-972f-449228f06d9f, infoPort=37781, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=2098193195;c=1732146006291), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T23:40:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5b7a408380e75e22 with lease ID 0x70efc4c9f27e94d0: Processing first storage report for DS-69b01ea0-df90-418e-9e52-6efc35b0f0d4 from datanode DatanodeRegistration(127.0.0.1:35447, datanodeUuid=942ccc57-68d5-4f40-972f-449228f06d9f, infoPort=37781, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=2098193195;c=1732146006291) 2024-11-20T23:40:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5b7a408380e75e22 with lease ID 0x70efc4c9f27e94d0: from storage DS-69b01ea0-df90-418e-9e52-6efc35b0f0d4 node DatanodeRegistration(127.0.0.1:35447, datanodeUuid=942ccc57-68d5-4f40-972f-449228f06d9f, infoPort=37781, infoSecurePort=0, ipcPort=44633, storageInfo=lv=-57;cid=testClusterID;nsid=2098193195;c=1732146006291), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T23:40:09,290 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5 2024-11-20T23:40:09,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/zookeeper_0, clientPort=53297, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T23:40:09,369 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53297 2024-11-20T23:40:09,378 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:09,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741825_1001 (size=7) 2024-11-20T23:40:09,984 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f with version=8 2024-11-20T23:40:09,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/hbase-staging 2024-11-20T23:40:10,077 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T23:40:10,302 INFO [Time-limited test {}] client.ConnectionUtils(128): master/412a5e44fd2e:0 server-side Connection retries=9 2024-11-20T23:40:10,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:10,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:10,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T23:40:10,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:10,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T23:40:10,511 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T23:40:10,574 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T23:40:10,588 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T23:40:10,593 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T23:40:10,626 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 127127 (auto-detected) 2024-11-20T23:40:10,627 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T23:40:10,649 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38443 2024-11-20T23:40:10,673 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38443 connecting to ZooKeeper ensemble=127.0.0.1:53297 2024-11-20T23:40:10,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384430x0, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T23:40:10,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38443-0x1015a9e90aa0000 connected 2024-11-20T23:40:10,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:10,891 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:10,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T23:40:10,908 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f, hbase.cluster.distributed=false 2024-11-20T23:40:10,931 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T23:40:10,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38443 2024-11-20T23:40:10,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38443 2024-11-20T23:40:10,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38443 2024-11-20T23:40:10,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38443 2024-11-20T23:40:10,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38443 2024-11-20T23:40:11,064 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/412a5e44fd2e:0 server-side Connection retries=9 2024-11-20T23:40:11,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:11,067 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:11,068 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T23:40:11,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T23:40:11,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T23:40:11,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T23:40:11,076 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T23:40:11,077 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32785 2024-11-20T23:40:11,079 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32785 connecting to ZooKeeper ensemble=127.0.0.1:53297 2024-11-20T23:40:11,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:11,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:11,106 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:327850x0, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T23:40:11,107 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32785-0x1015a9e90aa0001 connected 2024-11-20T23:40:11,108 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T23:40:11,112 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T23:40:11,119 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T23:40:11,122 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T23:40:11,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T23:40:11,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32785 2024-11-20T23:40:11,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32785 2024-11-20T23:40:11,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32785 2024-11-20T23:40:11,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32785 2024-11-20T23:40:11,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32785 2024-11-20T23:40:11,153 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;412a5e44fd2e:38443 2024-11-20T23:40:11,154 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:11,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T23:40:11,201 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T23:40:11,203 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:11,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:11,232 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T23:40:11,232 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:11,233 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T23:40:11,234 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/412a5e44fd2e,38443,1732146010136 from backup master directory 2024-11-20T23:40:11,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:11,242 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T23:40:11,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T23:40:11,243 WARN [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T23:40:11,243 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:11,245 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T23:40:11,247 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T23:40:11,307 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/hbase.id] with ID: db0ca5a0-cc6c-4110-ab8c-c571acd56fb9 2024-11-20T23:40:11,307 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/.tmp/hbase.id 2024-11-20T23:40:11,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741826_1002 (size=42) 2024-11-20T23:40:11,728 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/.tmp/hbase.id]:[hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/hbase.id] 2024-11-20T23:40:11,780 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T23:40:11,786 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T23:40:11,805 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T23:40:11,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:11,819 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:11,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741827_1003 (size=196) 2024-11-20T23:40:12,259 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:40:12,261 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T23:40:12,275 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T23:40:12,279 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T23:40:12,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741828_1004 (size=1189) 2024-11-20T23:40:12,734 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store 2024-11-20T23:40:12,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741829_1005 (size=34) 2024-11-20T23:40:13,164 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T23:40:13,167 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:13,168 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T23:40:13,168 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:40:13,168 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:40:13,170 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T23:40:13,170 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:40:13,170 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:40:13,171 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732146013168Disabling compacts and flushes for region at 1732146013168Disabling writes for close at 1732146013170 (+2 ms)Writing region close event to WAL at 1732146013170Closed at 1732146013170 2024-11-20T23:40:13,173 WARN [master/412a5e44fd2e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/.initializing 2024-11-20T23:40:13,174 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/WALs/412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:13,182 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T23:40:13,195 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=412a5e44fd2e%2C38443%2C1732146010136, suffix=, logDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/WALs/412a5e44fd2e,38443,1732146010136, archiveDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/oldWALs, maxLogs=10 2024-11-20T23:40:13,217 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/WALs/412a5e44fd2e,38443,1732146010136/412a5e44fd2e%2C38443%2C1732146010136.1732146013199, exclude list is [], retry=0 2024-11-20T23:40:13,233 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35447,DS-97311b13-b495-468d-ad3e-c04052ad0732,DISK] 2024-11-20T23:40:13,236 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T23:40:13,267 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/WALs/412a5e44fd2e,38443,1732146010136/412a5e44fd2e%2C38443%2C1732146010136.1732146013199 2024-11-20T23:40:13,268 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37781:37781)] 2024-11-20T23:40:13,269 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:40:13,269 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:13,272 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,273 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T23:40:13,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:13,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:13,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T23:40:13,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:13,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:13,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T23:40:13,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:13,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:13,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T23:40:13,353 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:13,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:13,355 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,359 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,360 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,365 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,366 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,370 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T23:40:13,374 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T23:40:13,379 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:40:13,380 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71772189, jitterRate=0.06948895752429962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T23:40:13,387 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732146013285Initializing all the Stores at 1732146013287 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146013288 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146013288Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146013289 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146013289Cleaning up temporary data from old regions at 1732146013366 (+77 ms)Region opened successfully at 1732146013387 (+21 ms) 2024-11-20T23:40:13,388 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T23:40:13,418 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e037e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=412a5e44fd2e/172.17.0.2:0 2024-11-20T23:40:13,445 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T23:40:13,454 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T23:40:13,455 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T23:40:13,457 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T23:40:13,458 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T23:40:13,462 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T23:40:13,463 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T23:40:13,488 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T23:40:13,496 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T23:40:13,558 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T23:40:13,560 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T23:40:13,562 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T23:40:13,568 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T23:40:13,570 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T23:40:13,574 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T23:40:13,579 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T23:40:13,580 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T23:40:13,589 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T23:40:13,609 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T23:40:13,621 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T23:40:13,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T23:40:13,632 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T23:40:13,632 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,636 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=412a5e44fd2e,38443,1732146010136, sessionid=0x1015a9e90aa0000, setting cluster-up flag (Was=false) 2024-11-20T23:40:13,674 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,705 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T23:40:13,707 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:13,726 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:13,758 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T23:40:13,760 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:13,768 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T23:40:13,834 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T23:40:13,843 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T23:40:13,846 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(746): ClusterId : db0ca5a0-cc6c-4110-ab8c-c571acd56fb9 2024-11-20T23:40:13,848 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T23:40:13,849 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T23:40:13,854 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 412a5e44fd2e,38443,1732146010136 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T23:40:13,863 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T23:40:13,863 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T23:40:13,864 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/412a5e44fd2e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T23:40:13,864 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/412a5e44fd2e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T23:40:13,865 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/412a5e44fd2e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T23:40:13,865 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/412a5e44fd2e:0, corePoolSize=5, maxPoolSize=5 2024-11-20T23:40:13,865 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/412a5e44fd2e:0, corePoolSize=10, maxPoolSize=10 2024-11-20T23:40:13,865 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:13,865 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/412a5e44fd2e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T23:40:13,866 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:13,867 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732146043867 2024-11-20T23:40:13,868 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T23:40:13,869 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T23:40:13,870 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T23:40:13,871 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T23:40:13,873 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T23:40:13,873 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T23:40:13,874 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T23:40:13,874 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T23:40:13,875 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T23:40:13,876 DEBUG [RS:0;412a5e44fd2e:32785 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71540463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=412a5e44fd2e/172.17.0.2:0 2024-11-20T23:40:13,875 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:13,877 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:13,877 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T23:40:13,879 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T23:40:13,881 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T23:40:13,882 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T23:40:13,885 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T23:40:13,885 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T23:40:13,887 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.large.0-1732146013886,5,FailOnTimeoutGroup] 2024-11-20T23:40:13,888 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.small.0-1732146013887,5,FailOnTimeoutGroup] 2024-11-20T23:40:13,888 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:13,888 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T23:40:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741831_1007 (size=1321) 2024-11-20T23:40:13,889 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:13,890 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:13,893 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;412a5e44fd2e:32785 2024-11-20T23:40:13,895 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T23:40:13,895 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T23:40:13,895 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T23:40:13,898 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(2659): reportForDuty to master=412a5e44fd2e,38443,1732146010136 with port=32785, startcode=1732146011021 2024-11-20T23:40:13,907 DEBUG [RS:0;412a5e44fd2e:32785 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T23:40:13,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49131, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T23:40:13,939 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38443 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:13,941 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38443 {}] master.ServerManager(517): Registering regionserver=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:13,954 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:40:13,954 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35501 2024-11-20T23:40:13,954 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T23:40:13,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T23:40:13,969 DEBUG [RS:0;412a5e44fd2e:32785 {}] zookeeper.ZKUtil(111): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:13,969 WARN [RS:0;412a5e44fd2e:32785 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T23:40:13,970 INFO [RS:0;412a5e44fd2e:32785 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T23:40:13,970 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:13,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [412a5e44fd2e,32785,1732146011021] 2024-11-20T23:40:13,999 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T23:40:14,014 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T23:40:14,018 INFO [RS:0;412a5e44fd2e:32785 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T23:40:14,018 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,019 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T23:40:14,024 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T23:40:14,025 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/412a5e44fd2e:0, corePoolSize=2, maxPoolSize=2 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,026 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/412a5e44fd2e:0, corePoolSize=1, maxPoolSize=1 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/412a5e44fd2e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T23:40:14,027 DEBUG [RS:0;412a5e44fd2e:32785 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/412a5e44fd2e:0, corePoolSize=3, maxPoolSize=3 2024-11-20T23:40:14,028 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,028 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,028 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,028 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,029 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,029 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,32785,1732146011021-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T23:40:14,045 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T23:40:14,046 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,32785,1732146011021-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,047 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,047 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.Replication(171): 412a5e44fd2e,32785,1732146011021 started 2024-11-20T23:40:14,062 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:14,062 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1482): Serving as 412a5e44fd2e,32785,1732146011021, RpcServer on 412a5e44fd2e/172.17.0.2:32785, sessionid=0x1015a9e90aa0001 2024-11-20T23:40:14,063 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T23:40:14,063 DEBUG [RS:0;412a5e44fd2e:32785 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:14,063 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '412a5e44fd2e,32785,1732146011021' 2024-11-20T23:40:14,063 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T23:40:14,064 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T23:40:14,065 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T23:40:14,065 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T23:40:14,065 DEBUG [RS:0;412a5e44fd2e:32785 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:14,065 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '412a5e44fd2e,32785,1732146011021' 2024-11-20T23:40:14,066 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T23:40:14,066 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T23:40:14,067 DEBUG [RS:0;412a5e44fd2e:32785 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T23:40:14,067 INFO [RS:0;412a5e44fd2e:32785 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T23:40:14,067 INFO [RS:0;412a5e44fd2e:32785 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T23:40:14,292 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T23:40:14,293 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:40:14,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741832_1008 (size=32) 2024-11-20T23:40:14,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:14,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T23:40:14,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T23:40:14,720 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:14,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:14,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T23:40:14,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T23:40:14,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:14,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:14,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T23:40:14,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T23:40:14,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:14,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:14,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T23:40:14,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T23:40:14,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:14,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:14,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T23:40:14,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740 2024-11-20T23:40:14,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740 2024-11-20T23:40:14,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T23:40:14,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T23:40:14,740 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T23:40:14,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T23:40:14,747 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:40:14,748 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69704378, jitterRate=0.038676172494888306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T23:40:14,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732146014712Initializing all the Stores at 1732146014716 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146014716Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146014716Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146014716Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146014717 (+1 ms)Cleaning up temporary data from old regions at 1732146014739 (+22 ms)Region opened successfully at 1732146014751 (+12 ms) 2024-11-20T23:40:14,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T23:40:14,751 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T23:40:14,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T23:40:14,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T23:40:14,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T23:40:14,753 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T23:40:14,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732146014751Disabling compacts and flushes for region at 1732146014751Disabling writes for close at 1732146014751Writing region close event to WAL at 1732146014752 (+1 ms)Closed at 1732146014753 (+1 ms) 2024-11-20T23:40:14,756 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T23:40:14,757 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T23:40:14,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T23:40:14,770 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T23:40:14,772 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T23:40:14,923 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:15,924 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:16,926 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:17,927 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:18,928 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:19,928 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:20,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T23:40:20,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T23:40:20,929 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:21,930 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:22,930 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:23,931 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:24,932 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:25,933 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:26,934 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:27,935 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:28,935 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:29,936 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:30,937 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:31,938 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:32,939 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:33,939 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:34,940 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:35,941 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:36,941 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:37,942 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:38,943 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:39,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T23:40:39,944 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:40,945 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:41,945 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:42,950 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:43,951 WARN [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T23:40:44,071 INFO [RS:0;412a5e44fd2e:32785 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T23:40:44,075 INFO [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=412a5e44fd2e%2C32785%2C1732146011021, suffix=, logDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021, archiveDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/oldWALs, maxLogs=32 2024-11-20T23:40:44,091 DEBUG [RS:0;412a5e44fd2e:32785 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021/412a5e44fd2e%2C32785%2C1732146011021.1732146044077, exclude list is [], retry=0 2024-11-20T23:40:44,095 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35447,DS-97311b13-b495-468d-ad3e-c04052ad0732,DISK] 2024-11-20T23:40:44,098 INFO [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021/412a5e44fd2e%2C32785%2C1732146011021.1732146044077 2024-11-20T23:40:44,099 DEBUG [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37781:37781)] 2024-11-20T23:40:44,204 DEBUG [412a5e44fd2e:38443 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T23:40:44,221 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:44,238 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 412a5e44fd2e,32785,1732146011021, state=OPENING 2024-11-20T23:40:44,251 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T23:40:44,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:44,264 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:40:44,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T23:40:44,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T23:40:44,273 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T23:40:44,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:40:44,477 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T23:40:44,481 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40287, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T23:40:44,497 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T23:40:44,498 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T23:40:44,498 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T23:40:44,504 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=412a5e44fd2e%2C32785%2C1732146011021.meta, suffix=.meta, logDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021, archiveDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/oldWALs, maxLogs=32 2024-11-20T23:40:44,529 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021/412a5e44fd2e%2C32785%2C1732146011021.meta.1732146044507.meta, exclude list is [], retry=0 2024-11-20T23:40:44,535 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35447,DS-97311b13-b495-468d-ad3e-c04052ad0732,DISK] 2024-11-20T23:40:44,540 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021/412a5e44fd2e%2C32785%2C1732146011021.meta.1732146044507.meta 2024-11-20T23:40:44,545 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37781:37781)] 2024-11-20T23:40:44,545 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:40:44,548 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T23:40:44,551 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T23:40:44,558 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T23:40:44,566 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T23:40:44,567 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:44,568 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T23:40:44,568 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T23:40:44,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T23:40:44,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T23:40:44,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:44,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:44,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T23:40:44,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T23:40:44,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:44,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:44,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T23:40:44,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T23:40:44,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:44,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:44,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T23:40:44,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T23:40:44,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:44,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T23:40:44,599 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T23:40:44,602 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740 2024-11-20T23:40:44,606 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740 2024-11-20T23:40:44,610 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T23:40:44,610 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T23:40:44,611 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T23:40:44,614 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T23:40:44,616 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69312086, jitterRate=0.03283056616783142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T23:40:44,617 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T23:40:44,619 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732146044569Writing region info on filesystem at 1732146044569Initializing all the Stores at 1732146044572 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146044572Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146044576 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146044576Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732146044576Cleaning up temporary data from old regions at 1732146044610 (+34 ms)Running coprocessor post-open hooks at 1732146044617 (+7 ms)Region opened successfully at 1732146044618 (+1 ms) 2024-11-20T23:40:44,628 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732146044466 2024-11-20T23:40:44,641 DEBUG [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T23:40:44,642 INFO [RS_OPEN_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T23:40:44,644 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:44,647 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 412a5e44fd2e,32785,1732146011021, state=OPEN 2024-11-20T23:40:44,699 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T23:40:44,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T23:40:44,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T23:40:44,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T23:40:44,701 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:44,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T23:40:44,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=412a5e44fd2e,32785,1732146011021 in 426 msec 2024-11-20T23:40:44,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T23:40:44,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.9520 sec 2024-11-20T23:40:44,729 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T23:40:44,729 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T23:40:44,756 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T23:40:44,758 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=412a5e44fd2e,32785,1732146011021, seqNum=-1] 2024-11-20T23:40:44,795 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T23:40:44,800 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51275, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T23:40:44,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0350 sec 2024-11-20T23:40:44,835 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732146044834, completionTime=-1 2024-11-20T23:40:44,837 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T23:40:44,838 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T23:40:44,876 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-20T23:40:44,876 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732146104876 2024-11-20T23:40:44,876 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732146164876 2024-11-20T23:40:44,876 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-20T23:40:44,879 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,879 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,880 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,881 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-412a5e44fd2e:38443, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,882 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,892 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,898 DEBUG [master/412a5e44fd2e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T23:40:44,929 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.685sec 2024-11-20T23:40:44,930 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T23:40:44,932 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T23:40:44,933 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T23:40:44,934 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T23:40:44,934 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T23:40:44,935 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T23:40:44,935 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T23:40:44,946 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T23:40:44,947 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T23:40:44,948 INFO [master/412a5e44fd2e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=412a5e44fd2e,38443,1732146010136-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T23:40:44,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,000 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T23:40:45,001 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T23:40:45,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 412a5e44fd2e,38443,-1 for getting cluster id 2024-11-20T23:40:45,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T23:40:45,117 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'db0ca5a0-cc6c-4110-ab8c-c571acd56fb9' 2024-11-20T23:40:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T23:40:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "db0ca5a0-cc6c-4110-ab8c-c571acd56fb9" 2024-11-20T23:40:45,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18214de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [412a5e44fd2e,38443,-1] 2024-11-20T23:40:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T23:40:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:40:45,156 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T23:40:45,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ba9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,161 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T23:40:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=412a5e44fd2e,32785,1732146011021, seqNum=-1] 2024-11-20T23:40:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T23:40:45,178 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T23:40:45,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:45,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14688263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 412a5e44fd2e,38443,-1 for getting cluster id 2024-11-20T23:40:45,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T23:40:45,216 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'db0ca5a0-cc6c-4110-ab8c-c571acd56fb9' 2024-11-20T23:40:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T23:40:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "db0ca5a0-cc6c-4110-ab8c-c571acd56fb9" 2024-11-20T23:40:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2adcb0ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [412a5e44fd2e,38443,-1] 2024-11-20T23:40:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T23:40:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:40:45,225 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T23:40:45,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a188c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T23:40:45,253 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=2632 2024-11-20T23:40:45,263 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T23:40:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 412a5e44fd2e,38443,1732146010136 2024-11-20T23:40:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@494ab692 2024-11-20T23:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T23:40:45,287 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T23:40:45,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:40:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T23:40:45,315 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:40:45,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-20T23:40:45,318 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:45,322 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:40:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T23:40:45,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741835_1011 (size=428) 2024-11-20T23:40:45,747 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f341086aa6d427ae5e5e95a7a4ed0d0, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:40:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741836_1012 (size=79) 2024-11-20T23:40:46,159 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:46,160 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9f341086aa6d427ae5e5e95a7a4ed0d0, disabling compactions & flushes 2024-11-20T23:40:46,160 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,160 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,160 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. after waiting 0 ms 2024-11-20T23:40:46,160 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,160 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,160 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f341086aa6d427ae5e5e95a7a4ed0d0: Waiting for close lock at 1732146046159Disabling compacts and flushes for region at 1732146046159Disabling writes for close at 1732146046160 (+1 ms)Writing region close event to WAL at 1732146046160Closed at 1732146046160 2024-11-20T23:40:46,163 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:40:46,168 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732146046163"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146046163"}]},"ts":"1732146046163"} 2024-11-20T23:40:46,174 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:40:46,178 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:40:46,181 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146046178"}]},"ts":"1732146046178"} 2024-11-20T23:40:46,187 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-20T23:40:46,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9f341086aa6d427ae5e5e95a7a4ed0d0, ASSIGN}] 2024-11-20T23:40:46,196 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9f341086aa6d427ae5e5e95a7a4ed0d0, ASSIGN 2024-11-20T23:40:46,198 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9f341086aa6d427ae5e5e95a7a4ed0d0, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:40:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T23:40:46,350 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9f341086aa6d427ae5e5e95a7a4ed0d0, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:46,357 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9f341086aa6d427ae5e5e95a7a4ed0d0, ASSIGN because future has completed 2024-11-20T23:40:46,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f341086aa6d427ae5e5e95a7a4ed0d0, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:40:46,524 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,525 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f341086aa6d427ae5e5e95a7a4ed0d0, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:40:46,526 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,526 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:46,526 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,526 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,530 INFO [StoreOpener-9f341086aa6d427ae5e5e95a7a4ed0d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,533 INFO [StoreOpener-9f341086aa6d427ae5e5e95a7a4ed0d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f341086aa6d427ae5e5e95a7a4ed0d0 columnFamilyName testFamily 2024-11-20T23:40:46,533 DEBUG [StoreOpener-9f341086aa6d427ae5e5e95a7a4ed0d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:46,534 INFO [StoreOpener-9f341086aa6d427ae5e5e95a7a4ed0d0-1 {}] regionserver.HStore(327): Store=9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:46,535 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,537 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,538 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,539 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,539 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,543 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,547 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:40:46,549 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9f341086aa6d427ae5e5e95a7a4ed0d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63500887, jitterRate=-0.053763046860694885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:40:46,549 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:40:46,550 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9f341086aa6d427ae5e5e95a7a4ed0d0: Running coprocessor pre-open hook at 1732146046526Writing region info on filesystem at 1732146046526Initializing all the Stores at 1732146046529 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146046529Cleaning up temporary data from old regions at 1732146046539 (+10 ms)Running coprocessor post-open hooks at 1732146046549 (+10 ms)Region opened successfully at 1732146046550 (+1 ms) 2024-11-20T23:40:46,552 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0., pid=6, masterSystemTime=1732146046514 2024-11-20T23:40:46,556 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,556 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:40:46,557 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9f341086aa6d427ae5e5e95a7a4ed0d0, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:46,561 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f341086aa6d427ae5e5e95a7a4ed0d0, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:40:46,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T23:40:46,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f341086aa6d427ae5e5e95a7a4ed0d0, server=412a5e44fd2e,32785,1732146011021 in 205 msec 2024-11-20T23:40:46,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T23:40:46,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9f341086aa6d427ae5e5e95a7a4ed0d0, ASSIGN in 377 msec 2024-11-20T23:40:46,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:40:46,574 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146046573"}]},"ts":"1732146046573"} 2024-11-20T23:40:46,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-20T23:40:46,579 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:40:46,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2780 sec 2024-11-20T23:40:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T23:40:48,354 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-20T23:40:48,354 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-20T23:40:48,356 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:40:48,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-20T23:40:48,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:40:48,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-20T23:40:48,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T23:40:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=412a5e44fd2e,32785,1732146011021, seqNum=-1] 2024-11-20T23:40:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T23:40:48,374 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T23:40:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:40:48,395 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:40:48,399 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-20T23:40:48,404 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T23:40:48,408 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T23:40:48,410 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T23:40:48,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-83249527192027128 2024-11-20T23:40:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146051409, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:40:48,436 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T23:40:48,438 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T23:40:48,439 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:40:48,441 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:40:48,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-83249527192027126 2024-11-20T23:40:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146051441, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:40:48,447 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:40:48,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-83249527192027125 2024-11-20T23:40:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146051447, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:40:48,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:40:48,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-83249527192027124 2024-11-20T23:40:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146051452, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:40:48,457 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:40:48,468 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-661843222_22 at /127.0.0.1:40934 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 346), ProcessCount=11 (was 11), AvailableMemoryMB=2386 (was 2632) 2024-11-20T23:40:48,476 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=2385 2024-11-20T23:40:48,479 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:40:48,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-20T23:40:48,484 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:40:48,484 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:48,485 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-20T23:40:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T23:40:48,487 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:40:48,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741837_1013 (size=414) 2024-11-20T23:40:48,905 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0252bb9c4c7b2f32b61e2212a71858a1, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:40:48,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741838_1014 (size=65) 2024-11-20T23:40:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 0252bb9c4c7b2f32b61e2212a71858a1, disabling compactions & flushes 2024-11-20T23:40:49,340 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. after waiting 0 ms 2024-11-20T23:40:49,341 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,341 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,341 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0252bb9c4c7b2f32b61e2212a71858a1: Waiting for close lock at 1732146049340Disabling compacts and flushes for region at 1732146049340Disabling writes for close at 1732146049341 (+1 ms)Writing region close event to WAL at 1732146049341Closed at 1732146049341 2024-11-20T23:40:49,344 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:40:49,344 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732146049344"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146049344"}]},"ts":"1732146049344"} 2024-11-20T23:40:49,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:40:49,353 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:40:49,353 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146049353"}]},"ts":"1732146049353"} 2024-11-20T23:40:49,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T23:40:49,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0252bb9c4c7b2f32b61e2212a71858a1, ASSIGN}] 2024-11-20T23:40:49,362 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0252bb9c4c7b2f32b61e2212a71858a1, ASSIGN 2024-11-20T23:40:49,366 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0252bb9c4c7b2f32b61e2212a71858a1, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:40:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T23:40:49,519 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0252bb9c4c7b2f32b61e2212a71858a1, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:49,524 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0252bb9c4c7b2f32b61e2212a71858a1, ASSIGN because future has completed 2024-11-20T23:40:49,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0252bb9c4c7b2f32b61e2212a71858a1, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:40:49,701 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0252bb9c4c7b2f32b61e2212a71858a1, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:40:49,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:49,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,708 INFO [StoreOpener-0252bb9c4c7b2f32b61e2212a71858a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,712 INFO [StoreOpener-0252bb9c4c7b2f32b61e2212a71858a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0252bb9c4c7b2f32b61e2212a71858a1 columnFamilyName testFamily 2024-11-20T23:40:49,712 DEBUG [StoreOpener-0252bb9c4c7b2f32b61e2212a71858a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:49,714 INFO [StoreOpener-0252bb9c4c7b2f32b61e2212a71858a1-1 {}] regionserver.HStore(327): Store=0252bb9c4c7b2f32b61e2212a71858a1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:49,716 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,725 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,725 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,731 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:40:49,747 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 0252bb9c4c7b2f32b61e2212a71858a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62089400, jitterRate=-0.07479584217071533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:40:49,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:40:49,748 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0252bb9c4c7b2f32b61e2212a71858a1: Running coprocessor pre-open hook at 1732146049703Writing region info on filesystem at 1732146049703Initializing all the Stores at 1732146049706 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146049706Cleaning up temporary data from old regions at 1732146049726 (+20 ms)Running coprocessor post-open hooks at 1732146049747 (+21 ms)Region opened successfully at 1732146049748 (+1 ms) 2024-11-20T23:40:49,750 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., pid=9, masterSystemTime=1732146049682 2024-11-20T23:40:49,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:40:49,759 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0252bb9c4c7b2f32b61e2212a71858a1, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:49,764 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0252bb9c4c7b2f32b61e2212a71858a1, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:40:49,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T23:40:49,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0252bb9c4c7b2f32b61e2212a71858a1, server=412a5e44fd2e,32785,1732146011021 in 242 msec 2024-11-20T23:40:49,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T23:40:49,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0252bb9c4c7b2f32b61e2212a71858a1, ASSIGN in 417 msec 2024-11-20T23:40:49,783 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:40:49,784 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146049783"}]},"ts":"1732146049783"} 2024-11-20T23:40:49,787 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T23:40:49,789 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:40:49,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.3090 sec 2024-11-20T23:40:50,120 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:40:50,164 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-20T23:40:50,166 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T23:40:50,167 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-20T23:40:50,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T23:40:50,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T23:40:50,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T23:40:50,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T23:40:50,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T23:40:50,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T23:40:50,571 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-20T23:40:50,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T23:40:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T23:40:51,518 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-20T23:40:51,518 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T23:40:51,518 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:40:51,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T23:40:51,525 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:40:51,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-20T23:40:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:40:51,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:40:51,574 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T23:40:51,580 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T23:40:51,583 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T23:40:55,206 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:40:57,604 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-20T23:40:57,605 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-20T23:40:57,606 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:40:57,618 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 346) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2022 (was 2385) 2024-11-20T23:40:57,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=2021 2024-11-20T23:40:57,640 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:40:57,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-20T23:40:57,645 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:40:57,645 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:57,645 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-20T23:40:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T23:40:57,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:40:57,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741839_1015 (size=412) 2024-11-20T23:40:58,063 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96847a8f1dc149e4522cad8ea9883e5a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:40:58,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741840_1016 (size=63) 2024-11-20T23:40:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-20T23:40:58,476 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:58,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 96847a8f1dc149e4522cad8ea9883e5a, disabling compactions & flushes 2024-11-20T23:40:58,477 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. after waiting 0 ms 2024-11-20T23:40:58,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,477 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,477 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96847a8f1dc149e4522cad8ea9883e5a: Waiting for close lock at 1732146058477Disabling compacts and flushes for region at 1732146058477Disabling writes for close at 1732146058477Writing region close event to WAL at 1732146058477Closed at 1732146058477 2024-11-20T23:40:58,479 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:40:58,480 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732146058480"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146058480"}]},"ts":"1732146058480"} 2024-11-20T23:40:58,484 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:40:58,486 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:40:58,487 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146058486"}]},"ts":"1732146058486"} 2024-11-20T23:40:58,490 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T23:40:58,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=96847a8f1dc149e4522cad8ea9883e5a, ASSIGN}] 2024-11-20T23:40:58,493 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=96847a8f1dc149e4522cad8ea9883e5a, ASSIGN 2024-11-20T23:40:58,494 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=96847a8f1dc149e4522cad8ea9883e5a, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:40:58,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=96847a8f1dc149e4522cad8ea9883e5a, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:58,650 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=96847a8f1dc149e4522cad8ea9883e5a, ASSIGN because future has completed 2024-11-20T23:40:58,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96847a8f1dc149e4522cad8ea9883e5a, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:40:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T23:40:58,815 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 96847a8f1dc149e4522cad8ea9883e5a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:40:58,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:40:58,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,819 INFO [StoreOpener-96847a8f1dc149e4522cad8ea9883e5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,822 INFO [StoreOpener-96847a8f1dc149e4522cad8ea9883e5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96847a8f1dc149e4522cad8ea9883e5a columnFamilyName testFamily 2024-11-20T23:40:58,822 DEBUG [StoreOpener-96847a8f1dc149e4522cad8ea9883e5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:40:58,823 INFO [StoreOpener-96847a8f1dc149e4522cad8ea9883e5a-1 {}] regionserver.HStore(327): Store=96847a8f1dc149e4522cad8ea9883e5a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:40:58,823 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,828 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,832 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:40:58,833 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 96847a8f1dc149e4522cad8ea9883e5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63427220, jitterRate=-0.05486077070236206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:40:58,833 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:40:58,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 96847a8f1dc149e4522cad8ea9883e5a: Running coprocessor pre-open hook at 1732146058816Writing region info on filesystem at 1732146058816Initializing all the Stores at 1732146058818 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146058819 (+1 ms)Cleaning up temporary data from old regions at 1732146058826 (+7 ms)Running coprocessor post-open hooks at 1732146058833 (+7 ms)Region opened successfully at 1732146058833 2024-11-20T23:40:58,835 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a., pid=12, masterSystemTime=1732146058806 2024-11-20T23:40:58,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,838 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:40:58,839 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=96847a8f1dc149e4522cad8ea9883e5a, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:40:58,841 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96847a8f1dc149e4522cad8ea9883e5a, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:40:58,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T23:40:58,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 96847a8f1dc149e4522cad8ea9883e5a, server=412a5e44fd2e,32785,1732146011021 in 192 msec 2024-11-20T23:40:58,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T23:40:58,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=96847a8f1dc149e4522cad8ea9883e5a, ASSIGN in 355 msec 2024-11-20T23:40:58,851 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:40:58,851 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146058851"}]},"ts":"1732146058851"} 2024-11-20T23:40:58,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T23:40:58,855 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:40:58,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2160 sec 2024-11-20T23:41:00,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-20T23:41:00,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T23:41:00,673 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-20T23:41:00,673 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T23:41:00,674 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:00,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T23:41:00,678 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:00,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-20T23:41:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:00,694 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:00,695 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T23:41:00,699 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T23:41:00,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T23:41:00,703 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T23:41:00,703 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T23:41:02,170 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-20T23:41:04,318 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:06,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027115' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T23:41:06,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:42972 deadline: 1732146072711, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027115' does not exist 2024-11-20T23:41:06,716 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 412a5e44fd2e,32785,1732146011021 for closing scanner id = -83249527192027115 for 96847a8f1dc149e4522cad8ea9883e5a of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027115' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T23:41:06,718 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=1908 (was 2021) 2024-11-20T23:41:06,726 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=1908 2024-11-20T23:41:06,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-20T23:41:06,732 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:06,733 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:06,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-20T23:41:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T23:41:06,735 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741841_1017 (size=423) 2024-11-20T23:41:07,147 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ee1609c7c5d894dcc55416f0d9960a5, NAME => 'testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:07,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741842_1018 (size=74) 2024-11-20T23:41:07,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-20T23:41:07,556 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:07,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 7ee1609c7c5d894dcc55416f0d9960a5, disabling compactions & flushes 2024-11-20T23:41:07,557 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. after waiting 0 ms 2024-11-20T23:41:07,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,557 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,557 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ee1609c7c5d894dcc55416f0d9960a5: Waiting for close lock at 1732146067557Disabling compacts and flushes for region at 1732146067557Disabling writes for close at 1732146067557Writing region close event to WAL at 1732146067557Closed at 1732146067557 2024-11-20T23:41:07,559 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:07,560 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732146067560"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146067560"}]},"ts":"1732146067560"} 2024-11-20T23:41:07,571 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:07,573 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:07,573 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146067573"}]},"ts":"1732146067573"} 2024-11-20T23:41:07,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-20T23:41:07,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7ee1609c7c5d894dcc55416f0d9960a5, ASSIGN}] 2024-11-20T23:41:07,581 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7ee1609c7c5d894dcc55416f0d9960a5, ASSIGN 2024-11-20T23:41:07,583 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7ee1609c7c5d894dcc55416f0d9960a5, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:07,735 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7ee1609c7c5d894dcc55416f0d9960a5, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:07,740 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7ee1609c7c5d894dcc55416f0d9960a5, ASSIGN because future has completed 2024-11-20T23:41:07,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee1609c7c5d894dcc55416f0d9960a5, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T23:41:07,904 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,905 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ee1609c7c5d894dcc55416f0d9960a5, NAME => 'testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:07,905 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,905 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:07,905 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,905 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,908 INFO [StoreOpener-7ee1609c7c5d894dcc55416f0d9960a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,912 INFO [StoreOpener-7ee1609c7c5d894dcc55416f0d9960a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ee1609c7c5d894dcc55416f0d9960a5 columnFamilyName testFamily 2024-11-20T23:41:07,913 DEBUG [StoreOpener-7ee1609c7c5d894dcc55416f0d9960a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:07,913 INFO [StoreOpener-7ee1609c7c5d894dcc55416f0d9960a5-1 {}] regionserver.HStore(327): Store=7ee1609c7c5d894dcc55416f0d9960a5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:07,914 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,915 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,916 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,917 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,917 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,920 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,925 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:07,925 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 7ee1609c7c5d894dcc55416f0d9960a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68527859, jitterRate=0.021144673228263855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:07,926 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:07,926 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 7ee1609c7c5d894dcc55416f0d9960a5: Running coprocessor pre-open hook at 1732146067906Writing region info on filesystem at 1732146067906Initializing all the Stores at 1732146067908 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146067908Cleaning up temporary data from old regions at 1732146067917 (+9 ms)Running coprocessor post-open hooks at 1732146067926 (+9 ms)Region opened successfully at 1732146067926 2024-11-20T23:41:07,928 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5., pid=15, masterSystemTime=1732146067897 2024-11-20T23:41:07,931 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,932 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:07,933 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7ee1609c7c5d894dcc55416f0d9960a5, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:07,936 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ee1609c7c5d894dcc55416f0d9960a5, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:07,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T23:41:07,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ee1609c7c5d894dcc55416f0d9960a5, server=412a5e44fd2e,32785,1732146011021 in 196 msec 2024-11-20T23:41:07,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T23:41:07,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7ee1609c7c5d894dcc55416f0d9960a5, ASSIGN in 363 msec 2024-11-20T23:41:07,946 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:07,947 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146067946"}]},"ts":"1732146067946"} 2024-11-20T23:41:07,949 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-20T23:41:07,951 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:07,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2220 sec 2024-11-20T23:41:08,173 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-20T23:41:09,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T23:41:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T23:41:09,774 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-20T23:41:09,774 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-20T23:41:09,774 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:09,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-20T23:41:09,781 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:09,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-20T23:41:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:09,802 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:09,803 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-20T23:41:09,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T23:41:09,810 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T23:41:09,812 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T23:41:09,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-83249527192027111 2024-11-20T23:41:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146072812, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:41:09,818 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T23:41:09,820 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T23:41:09,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:41:09,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:41:09,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-83249527192027109 2024-11-20T23:41:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146072822, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:41:09,827 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:41:09,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-83249527192027108 2024-11-20T23:41:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146072826, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:41:09,833 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:41:09,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-83249527192027107 2024-11-20T23:41:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:42972 deadline: 1732146072833, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T23:41:09,839 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T23:41:09,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 333), ProcessCount=11 (was 11), AvailableMemoryMB=1766 (was 1908) 2024-11-20T23:41:09,864 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=1764 2024-11-20T23:41:09,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-20T23:41:09,870 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:09,871 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:09,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-20T23:41:09,872 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T23:41:09,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741843_1019 (size=418) 2024-11-20T23:41:10,284 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 69bf8e54f5b7d477f3a5e23f1012952d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:10,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741844_1020 (size=69) 2024-11-20T23:41:10,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-20T23:41:10,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:10,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:10,696 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 69bf8e54f5b7d477f3a5e23f1012952d, disabling compactions & flushes 2024-11-20T23:41:10,696 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:10,696 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:10,696 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. after waiting 0 ms 2024-11-20T23:41:10,696 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:10,696 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:10,696 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 69bf8e54f5b7d477f3a5e23f1012952d: Waiting for close lock at 1732146070696Disabling compacts and flushes for region at 1732146070696Disabling writes for close at 1732146070696Writing region close event to WAL at 1732146070696Closed at 1732146070696 2024-11-20T23:41:10,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:10,699 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732146070698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146070698"}]},"ts":"1732146070698"} 2024-11-20T23:41:10,702 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:10,704 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:10,705 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146070705"}]},"ts":"1732146070705"} 2024-11-20T23:41:10,708 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T23:41:10,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=69bf8e54f5b7d477f3a5e23f1012952d, ASSIGN}] 2024-11-20T23:41:10,711 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=69bf8e54f5b7d477f3a5e23f1012952d, ASSIGN 2024-11-20T23:41:10,712 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=69bf8e54f5b7d477f3a5e23f1012952d, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:10,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=69bf8e54f5b7d477f3a5e23f1012952d, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:10,867 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=69bf8e54f5b7d477f3a5e23f1012952d, ASSIGN because future has completed 2024-11-20T23:41:10,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69bf8e54f5b7d477f3a5e23f1012952d, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T23:41:11,028 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:11,028 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 69bf8e54f5b7d477f3a5e23f1012952d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:11,029 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,029 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:11,029 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,029 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,032 INFO [StoreOpener-69bf8e54f5b7d477f3a5e23f1012952d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,035 INFO [StoreOpener-69bf8e54f5b7d477f3a5e23f1012952d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69bf8e54f5b7d477f3a5e23f1012952d columnFamilyName testFamily 2024-11-20T23:41:11,035 DEBUG [StoreOpener-69bf8e54f5b7d477f3a5e23f1012952d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:11,036 INFO [StoreOpener-69bf8e54f5b7d477f3a5e23f1012952d-1 {}] regionserver.HStore(327): Store=69bf8e54f5b7d477f3a5e23f1012952d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:11,036 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,037 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,038 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,038 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,038 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,041 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,044 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:11,045 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 69bf8e54f5b7d477f3a5e23f1012952d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69297091, jitterRate=0.032607123255729675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:11,045 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:11,046 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 69bf8e54f5b7d477f3a5e23f1012952d: Running coprocessor pre-open hook at 1732146071029Writing region info on filesystem at 1732146071029Initializing all the Stores at 1732146071031 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146071031Cleaning up temporary data from old regions at 1732146071038 (+7 ms)Running coprocessor post-open hooks at 1732146071045 (+7 ms)Region opened successfully at 1732146071046 (+1 ms) 2024-11-20T23:41:11,047 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., pid=18, masterSystemTime=1732146071022 2024-11-20T23:41:11,050 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:11,050 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:11,051 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=69bf8e54f5b7d477f3a5e23f1012952d, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:11,054 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69bf8e54f5b7d477f3a5e23f1012952d, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:11,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T23:41:11,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 69bf8e54f5b7d477f3a5e23f1012952d, server=412a5e44fd2e,32785,1732146011021 in 189 msec 2024-11-20T23:41:11,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T23:41:11,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=69bf8e54f5b7d477f3a5e23f1012952d, ASSIGN in 350 msec 2024-11-20T23:41:11,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:11,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146071065"}]},"ts":"1732146071065"} 2024-11-20T23:41:11,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T23:41:11,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:11,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2030 sec 2024-11-20T23:41:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T23:41:12,893 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-20T23:41:12,893 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T23:41:12,893 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:12,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T23:41:12,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:12,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-20T23:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:12,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:12,920 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T23:41:12,924 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T23:41:12,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T23:41:13,407 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:13,927 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T23:41:13,927 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T23:41:13,927 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:41:14,174 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-20T23:41:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-20T23:41:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:14,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T23:41:15,927 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-20T23:41:15,928 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-20T23:41:15,928 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:41:15,938 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=1605 (was 1764) 2024-11-20T23:41:15,947 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=1605 2024-11-20T23:41:15,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-20T23:41:15,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:15,953 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:15,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-20T23:41:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T23:41:15,954 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:15,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741845_1021 (size=416) 2024-11-20T23:41:16,367 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 428b56542c01920361d3b108ed2e6dd4, NAME => 'testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:16,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741846_1022 (size=67) 2024-11-20T23:41:16,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-20T23:41:16,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:16,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 428b56542c01920361d3b108ed2e6dd4, disabling compactions & flushes 2024-11-20T23:41:16,782 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:16,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:16,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. after waiting 0 ms 2024-11-20T23:41:16,782 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:16,782 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:16,783 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 428b56542c01920361d3b108ed2e6dd4: Waiting for close lock at 1732146076782Disabling compacts and flushes for region at 1732146076782Disabling writes for close at 1732146076782Writing region close event to WAL at 1732146076782Closed at 1732146076782 2024-11-20T23:41:16,785 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:16,786 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732146076785"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146076785"}]},"ts":"1732146076785"} 2024-11-20T23:41:16,790 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:16,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:16,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146076792"}]},"ts":"1732146076792"} 2024-11-20T23:41:16,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T23:41:16,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=428b56542c01920361d3b108ed2e6dd4, ASSIGN}] 2024-11-20T23:41:16,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=428b56542c01920361d3b108ed2e6dd4, ASSIGN 2024-11-20T23:41:16,799 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=428b56542c01920361d3b108ed2e6dd4, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:16,951 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=428b56542c01920361d3b108ed2e6dd4, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:16,960 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=428b56542c01920361d3b108ed2e6dd4, ASSIGN because future has completed 2024-11-20T23:41:16,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 428b56542c01920361d3b108ed2e6dd4, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T23:41:17,122 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:17,122 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 428b56542c01920361d3b108ed2e6dd4, NAME => 'testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:17,123 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,123 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:17,123 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,123 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,126 INFO [StoreOpener-428b56542c01920361d3b108ed2e6dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,129 INFO [StoreOpener-428b56542c01920361d3b108ed2e6dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428b56542c01920361d3b108ed2e6dd4 columnFamilyName testFamily 2024-11-20T23:41:17,129 DEBUG [StoreOpener-428b56542c01920361d3b108ed2e6dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:17,130 INFO [StoreOpener-428b56542c01920361d3b108ed2e6dd4-1 {}] regionserver.HStore(327): Store=428b56542c01920361d3b108ed2e6dd4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:17,130 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,131 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,132 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,133 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,133 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,135 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,138 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:17,138 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 428b56542c01920361d3b108ed2e6dd4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67558482, jitterRate=0.0066998302936553955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:17,138 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:17,139 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 428b56542c01920361d3b108ed2e6dd4: Running coprocessor pre-open hook at 1732146077124Writing region info on filesystem at 1732146077124Initializing all the Stores at 1732146077126 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146077126Cleaning up temporary data from old regions at 1732146077133 (+7 ms)Running coprocessor post-open hooks at 1732146077138 (+5 ms)Region opened successfully at 1732146077139 (+1 ms) 2024-11-20T23:41:17,140 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4., pid=21, masterSystemTime=1732146077114 2024-11-20T23:41:17,143 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:17,143 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:17,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=428b56542c01920361d3b108ed2e6dd4, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:17,146 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 428b56542c01920361d3b108ed2e6dd4, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:17,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T23:41:17,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 428b56542c01920361d3b108ed2e6dd4, server=412a5e44fd2e,32785,1732146011021 in 187 msec 2024-11-20T23:41:17,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T23:41:17,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=428b56542c01920361d3b108ed2e6dd4, ASSIGN in 356 msec 2024-11-20T23:41:17,155 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:17,155 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146077155"}]},"ts":"1732146077155"} 2024-11-20T23:41:17,158 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T23:41:17,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:17,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2100 sec 2024-11-20T23:41:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T23:41:18,983 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T23:41:18,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-20T23:41:18,984 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:18,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T23:41:18,989 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:18,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-20T23:41:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:19,007 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:19,007 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T23:41:19,011 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T23:41:19,014 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T23:41:19,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T23:41:19,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T23:41:20,176 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-20T23:41:20,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-20T23:41:20,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:20,571 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-20T23:41:20,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:22,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32785 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027096' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T23:41:22,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32785 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:42972 deadline: 1732146083018, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027096' does not exist 2024-11-20T23:41:22,021 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 412a5e44fd2e,32785,1732146011021 for closing scanner id = -83249527192027096 for 428b56542c01920361d3b108ed2e6dd4 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027096' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T23:41:22,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 297), ProcessCount=11 (was 11), AvailableMemoryMB=1512 (was 1605) 2024-11-20T23:41:22,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=1511 2024-11-20T23:41:22,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-20T23:41:22,039 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:22,039 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:22,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-20T23:41:22,040 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T23:41:22,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741847_1023 (size=411) 2024-11-20T23:41:22,455 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 71c6fe5965cb00c238e2d6f3bcc565d1, NAME => 'testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:22,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741848_1024 (size=62) 2024-11-20T23:41:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-20T23:41:22,625 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:22,870 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:22,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 71c6fe5965cb00c238e2d6f3bcc565d1, disabling compactions & flushes 2024-11-20T23:41:22,871 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:22,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:22,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. after waiting 0 ms 2024-11-20T23:41:22,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:22,871 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:22,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 71c6fe5965cb00c238e2d6f3bcc565d1: Waiting for close lock at 1732146082871Disabling compacts and flushes for region at 1732146082871Disabling writes for close at 1732146082871Writing region close event to WAL at 1732146082871Closed at 1732146082871 2024-11-20T23:41:22,874 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:22,875 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732146082875"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146082875"}]},"ts":"1732146082875"} 2024-11-20T23:41:22,880 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:22,882 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:22,883 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146082882"}]},"ts":"1732146082882"} 2024-11-20T23:41:22,887 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T23:41:22,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=71c6fe5965cb00c238e2d6f3bcc565d1, ASSIGN}] 2024-11-20T23:41:22,889 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=71c6fe5965cb00c238e2d6f3bcc565d1, ASSIGN 2024-11-20T23:41:22,891 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=71c6fe5965cb00c238e2d6f3bcc565d1, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:23,042 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=71c6fe5965cb00c238e2d6f3bcc565d1, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:23,044 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=71c6fe5965cb00c238e2d6f3bcc565d1, ASSIGN because future has completed 2024-11-20T23:41:23,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71c6fe5965cb00c238e2d6f3bcc565d1, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T23:41:23,208 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:23,209 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 71c6fe5965cb00c238e2d6f3bcc565d1, NAME => 'testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:23,209 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,209 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:23,209 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,209 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,212 INFO [StoreOpener-71c6fe5965cb00c238e2d6f3bcc565d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,214 INFO [StoreOpener-71c6fe5965cb00c238e2d6f3bcc565d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 71c6fe5965cb00c238e2d6f3bcc565d1 columnFamilyName testFamily 2024-11-20T23:41:23,214 DEBUG [StoreOpener-71c6fe5965cb00c238e2d6f3bcc565d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:23,215 INFO [StoreOpener-71c6fe5965cb00c238e2d6f3bcc565d1-1 {}] regionserver.HStore(327): Store=71c6fe5965cb00c238e2d6f3bcc565d1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:23,215 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,216 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,217 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,217 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,217 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,219 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,222 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:23,223 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 71c6fe5965cb00c238e2d6f3bcc565d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59385716, jitterRate=-0.11508387327194214}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:23,223 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:23,225 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 71c6fe5965cb00c238e2d6f3bcc565d1: Running coprocessor pre-open hook at 1732146083210Writing region info on filesystem at 1732146083210Initializing all the Stores at 1732146083211 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146083211Cleaning up temporary data from old regions at 1732146083217 (+6 ms)Running coprocessor post-open hooks at 1732146083223 (+6 ms)Region opened successfully at 1732146083225 (+2 ms) 2024-11-20T23:41:23,226 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1., pid=24, masterSystemTime=1732146083197 2024-11-20T23:41:23,228 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:23,229 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:23,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=71c6fe5965cb00c238e2d6f3bcc565d1, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:23,232 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 71c6fe5965cb00c238e2d6f3bcc565d1, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:23,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T23:41:23,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 71c6fe5965cb00c238e2d6f3bcc565d1, server=412a5e44fd2e,32785,1732146011021 in 188 msec 2024-11-20T23:41:23,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-20T23:41:23,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=71c6fe5965cb00c238e2d6f3bcc565d1, ASSIGN in 349 msec 2024-11-20T23:41:23,239 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:23,240 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146083240"}]},"ts":"1732146083240"} 2024-11-20T23:41:23,242 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T23:41:23,243 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:23,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2080 sec 2024-11-20T23:41:24,986 INFO [master/412a5e44fd2e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T23:41:24,986 INFO [master/412a5e44fd2e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T23:41:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T23:41:25,064 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-20T23:41:25,064 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T23:41:25,064 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:25,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T23:41:25,071 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:25,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-20T23:41:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:25,090 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:25,091 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T23:41:25,095 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T23:41:25,098 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T23:41:25,100 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T23:41:25,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T23:41:26,178 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-20T23:41:28,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T23:41:28,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:42972 deadline: 1732146089112, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027092' does not exist 2024-11-20T23:41:28,115 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 412a5e44fd2e,32785,1732146011021 for closing scanner id = -83249527192027092 for 71c6fe5965cb00c238e2d6f3bcc565d1 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T23:41:28,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1444 (was 1511) 2024-11-20T23:41:28,130 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=1444 2024-11-20T23:41:28,132 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:28,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T23:41:28,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:28,136 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:28,136 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-20T23:41:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T23:41:28,138 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:28,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741849_1025 (size=419) 2024-11-20T23:41:28,549 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 823bc8b5555cc1423e230ccf63c5f45a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:28,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741850_1026 (size=70) 2024-11-20T23:41:28,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-20T23:41:28,711 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:28,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:28,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 823bc8b5555cc1423e230ccf63c5f45a, disabling compactions & flushes 2024-11-20T23:41:28,959 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:28,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:28,960 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. after waiting 0 ms 2024-11-20T23:41:28,960 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:28,960 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:28,960 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 823bc8b5555cc1423e230ccf63c5f45a: Waiting for close lock at 1732146088959Disabling compacts and flushes for region at 1732146088959Disabling writes for close at 1732146088960 (+1 ms)Writing region close event to WAL at 1732146088960Closed at 1732146088960 2024-11-20T23:41:28,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:28,962 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732146088961"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146088961"}]},"ts":"1732146088961"} 2024-11-20T23:41:28,964 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:28,966 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:28,966 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146088966"}]},"ts":"1732146088966"} 2024-11-20T23:41:28,969 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T23:41:28,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=823bc8b5555cc1423e230ccf63c5f45a, ASSIGN}] 2024-11-20T23:41:28,970 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=823bc8b5555cc1423e230ccf63c5f45a, ASSIGN 2024-11-20T23:41:28,972 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=823bc8b5555cc1423e230ccf63c5f45a, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:29,122 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=823bc8b5555cc1423e230ccf63c5f45a, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:29,125 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=823bc8b5555cc1423e230ccf63c5f45a, ASSIGN because future has completed 2024-11-20T23:41:29,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 823bc8b5555cc1423e230ccf63c5f45a, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T23:41:29,289 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:29,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 823bc8b5555cc1423e230ccf63c5f45a, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:29,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:29,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,291 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,293 INFO [StoreOpener-823bc8b5555cc1423e230ccf63c5f45a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,296 INFO [StoreOpener-823bc8b5555cc1423e230ccf63c5f45a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 823bc8b5555cc1423e230ccf63c5f45a columnFamilyName testFamily 2024-11-20T23:41:29,296 DEBUG [StoreOpener-823bc8b5555cc1423e230ccf63c5f45a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:29,297 INFO [StoreOpener-823bc8b5555cc1423e230ccf63c5f45a-1 {}] regionserver.HStore(327): Store=823bc8b5555cc1423e230ccf63c5f45a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:29,297 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,298 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,299 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,304 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:29,305 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 823bc8b5555cc1423e230ccf63c5f45a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59598346, jitterRate=-0.11191543936729431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:29,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:29,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 823bc8b5555cc1423e230ccf63c5f45a: Running coprocessor pre-open hook at 1732146089291Writing region info on filesystem at 1732146089291Initializing all the Stores at 1732146089292 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146089292Cleaning up temporary data from old regions at 1732146089299 (+7 ms)Running coprocessor post-open hooks at 1732146089305 (+6 ms)Region opened successfully at 1732146089305 2024-11-20T23:41:29,307 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., pid=27, masterSystemTime=1732146089279 2024-11-20T23:41:29,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:29,311 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:29,312 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=823bc8b5555cc1423e230ccf63c5f45a, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:29,315 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 823bc8b5555cc1423e230ccf63c5f45a, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:29,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T23:41:29,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 823bc8b5555cc1423e230ccf63c5f45a, server=412a5e44fd2e,32785,1732146011021 in 191 msec 2024-11-20T23:41:29,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T23:41:29,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=823bc8b5555cc1423e230ccf63c5f45a, ASSIGN in 351 msec 2024-11-20T23:41:29,324 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:29,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146089324"}]},"ts":"1732146089324"} 2024-11-20T23:41:29,327 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T23:41:29,328 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:29,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1960 sec 2024-11-20T23:41:30,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-20T23:41:30,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:30,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T23:41:30,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T23:41:31,164 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-20T23:41:31,164 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T23:41:31,164 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:31,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T23:41:31,171 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:31,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T23:41:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:31,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:31,188 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T23:41:31,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T23:41:31,194 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T23:41:32,180 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-20T23:41:34,793 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:37,197 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-20T23:41:37,197 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-11-20T23:41:37,197 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:41:37,205 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 292), ProcessCount=11 (was 11), AvailableMemoryMB=1374 (was 1444) 2024-11-20T23:41:37,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=1373 2024-11-20T23:41:37,217 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:37,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-20T23:41:37,221 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:37,221 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:37,221 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-20T23:41:37,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T23:41:37,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741851_1027 (size=407) 2024-11-20T23:41:37,234 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3830156c857c64aa567f2601eaf4755e, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:37,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741852_1028 (size=58) 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 3830156c857c64aa567f2601eaf4755e, disabling compactions & flushes 2024-11-20T23:41:37,242 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. after waiting 0 ms 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,242 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,242 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3830156c857c64aa567f2601eaf4755e: Waiting for close lock at 1732146097242Disabling compacts and flushes for region at 1732146097242Disabling writes for close at 1732146097242Writing region close event to WAL at 1732146097242Closed at 1732146097242 2024-11-20T23:41:37,243 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:37,244 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732146097244"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146097244"}]},"ts":"1732146097244"} 2024-11-20T23:41:37,247 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:37,248 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:37,248 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146097248"}]},"ts":"1732146097248"} 2024-11-20T23:41:37,251 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T23:41:37,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3830156c857c64aa567f2601eaf4755e, ASSIGN}] 2024-11-20T23:41:37,254 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3830156c857c64aa567f2601eaf4755e, ASSIGN 2024-11-20T23:41:37,255 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3830156c857c64aa567f2601eaf4755e, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:37,406 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3830156c857c64aa567f2601eaf4755e, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:37,409 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3830156c857c64aa567f2601eaf4755e, ASSIGN because future has completed 2024-11-20T23:41:37,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3830156c857c64aa567f2601eaf4755e, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:37,574 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,574 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 3830156c857c64aa567f2601eaf4755e, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:37,574 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,574 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:37,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,576 INFO [StoreOpener-3830156c857c64aa567f2601eaf4755e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,578 INFO [StoreOpener-3830156c857c64aa567f2601eaf4755e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3830156c857c64aa567f2601eaf4755e columnFamilyName testFamily 2024-11-20T23:41:37,578 DEBUG [StoreOpener-3830156c857c64aa567f2601eaf4755e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:37,579 INFO [StoreOpener-3830156c857c64aa567f2601eaf4755e-1 {}] regionserver.HStore(327): Store=3830156c857c64aa567f2601eaf4755e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:37,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:37,587 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 3830156c857c64aa567f2601eaf4755e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66130923, jitterRate=-0.01457245647907257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:37,587 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:37,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 3830156c857c64aa567f2601eaf4755e: Running coprocessor pre-open hook at 1732146097575Writing region info on filesystem at 1732146097575Initializing all the Stores at 1732146097576 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146097576Cleaning up temporary data from old regions at 1732146097581 (+5 ms)Running coprocessor post-open hooks at 1732146097587 (+6 ms)Region opened successfully at 1732146097587 2024-11-20T23:41:37,589 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e., pid=30, masterSystemTime=1732146097563 2024-11-20T23:41:37,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,592 INFO [RS_OPEN_PRIORITY_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:37,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=3830156c857c64aa567f2601eaf4755e, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:37,596 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3830156c857c64aa567f2601eaf4755e, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:37,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T23:41:37,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 3830156c857c64aa567f2601eaf4755e, server=412a5e44fd2e,32785,1732146011021 in 189 msec 2024-11-20T23:41:37,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-20T23:41:37,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=3830156c857c64aa567f2601eaf4755e, ASSIGN in 350 msec 2024-11-20T23:41:37,606 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:37,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146097606"}]},"ts":"1732146097606"} 2024-11-20T23:41:37,609 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T23:41:37,610 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:37,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 393 msec 2024-11-20T23:41:37,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-20T23:41:38,184 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-20T23:41:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T23:41:38,233 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-20T23:41:38,233 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T23:41:38,234 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:38,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T23:41:38,239 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:38,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-20T23:41:38,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:38,253 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:38,253 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T23:41:38,256 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T23:41:38,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T23:41:38,259 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T23:41:38,260 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T23:41:39,287 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T23:41:40,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-20T23:41:40,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:43,078 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:44,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027084' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T23:41:44,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32785 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:42972 deadline: 1732146110269, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027084' does not exist 2024-11-20T23:41:44,270 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 412a5e44fd2e,32785,1732146011021 for closing scanner id = -83249527192027084 for 3830156c857c64aa567f2601eaf4755e of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-83249527192027084' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T23:41:44,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=1318 (was 1373) 2024-11-20T23:41:44,282 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=1318 2024-11-20T23:41:44,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T23:41:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-20T23:41:44,287 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T23:41:44,288 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:44,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-20T23:41:44,289 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T23:41:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T23:41:44,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741853_1029 (size=423) 2024-11-20T23:41:44,698 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33a2ef971a60bc25e4ac3d3f03f5d530, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f 2024-11-20T23:41:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741854_1030 (size=74) 2024-11-20T23:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-20T23:41:44,895 DEBUG [master/412a5e44fd2e:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 33a2ef971a60bc25e4ac3d3f03f5d530, disabling compactions & flushes 2024-11-20T23:41:45,107 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. after waiting 0 ms 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,107 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,107 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33a2ef971a60bc25e4ac3d3f03f5d530: Waiting for close lock at 1732146105107Disabling compacts and flushes for region at 1732146105107Disabling writes for close at 1732146105107Writing region close event to WAL at 1732146105107Closed at 1732146105107 2024-11-20T23:41:45,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T23:41:45,110 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732146105109"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732146105109"}]},"ts":"1732146105109"} 2024-11-20T23:41:45,113 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T23:41:45,114 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T23:41:45,115 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146105115"}]},"ts":"1732146105115"} 2024-11-20T23:41:45,118 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T23:41:45,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=33a2ef971a60bc25e4ac3d3f03f5d530, ASSIGN}] 2024-11-20T23:41:45,120 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=33a2ef971a60bc25e4ac3d3f03f5d530, ASSIGN 2024-11-20T23:41:45,121 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=33a2ef971a60bc25e4ac3d3f03f5d530, ASSIGN; state=OFFLINE, location=412a5e44fd2e,32785,1732146011021; forceNewPlan=false, retain=false 2024-11-20T23:41:45,272 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=33a2ef971a60bc25e4ac3d3f03f5d530, regionState=OPENING, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:45,275 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=33a2ef971a60bc25e4ac3d3f03f5d530, ASSIGN because future has completed 2024-11-20T23:41:45,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33a2ef971a60bc25e4ac3d3f03f5d530, server=412a5e44fd2e,32785,1732146011021}] 2024-11-20T23:41:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T23:41:45,439 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,440 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 33a2ef971a60bc25e4ac3d3f03f5d530, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530.', STARTKEY => '', ENDKEY => ''} 2024-11-20T23:41:45,440 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,440 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T23:41:45,440 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,440 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,442 INFO [StoreOpener-33a2ef971a60bc25e4ac3d3f03f5d530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,445 INFO [StoreOpener-33a2ef971a60bc25e4ac3d3f03f5d530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33a2ef971a60bc25e4ac3d3f03f5d530 columnFamilyName testFamily 2024-11-20T23:41:45,445 DEBUG [StoreOpener-33a2ef971a60bc25e4ac3d3f03f5d530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T23:41:45,445 INFO [StoreOpener-33a2ef971a60bc25e4ac3d3f03f5d530-1 {}] regionserver.HStore(327): Store=33a2ef971a60bc25e4ac3d3f03f5d530/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T23:41:45,446 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,447 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,447 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,448 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,448 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,451 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,454 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T23:41:45,455 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 33a2ef971a60bc25e4ac3d3f03f5d530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59660422, jitterRate=-0.11099043488502502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T23:41:45,455 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:45,456 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 33a2ef971a60bc25e4ac3d3f03f5d530: Running coprocessor pre-open hook at 1732146105440Writing region info on filesystem at 1732146105440Initializing all the Stores at 1732146105442 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732146105442Cleaning up temporary data from old regions at 1732146105448 (+6 ms)Running coprocessor post-open hooks at 1732146105455 (+7 ms)Region opened successfully at 1732146105456 (+1 ms) 2024-11-20T23:41:45,457 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., pid=33, masterSystemTime=1732146105428 2024-11-20T23:41:45,460 DEBUG [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,460 INFO [RS_OPEN_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:45,461 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=33a2ef971a60bc25e4ac3d3f03f5d530, regionState=OPEN, openSeqNum=2, regionLocation=412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:45,463 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33a2ef971a60bc25e4ac3d3f03f5d530, server=412a5e44fd2e,32785,1732146011021 because future has completed 2024-11-20T23:41:45,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-20T23:41:45,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 33a2ef971a60bc25e4ac3d3f03f5d530, server=412a5e44fd2e,32785,1732146011021 in 188 msec 2024-11-20T23:41:45,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T23:41:45,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=33a2ef971a60bc25e4ac3d3f03f5d530, ASSIGN in 349 msec 2024-11-20T23:41:45,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T23:41:45,473 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732146105472"}]},"ts":"1732146105472"} 2024-11-20T23:41:45,475 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T23:41:45,477 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T23:41:45,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1930 sec 2024-11-20T23:41:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T23:41:47,324 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-20T23:41:47,324 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T23:41:47,324 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:47,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T23:41:47,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T23:41:47,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T23:41:47,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:47,344 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T23:41:47,345 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T23:41:47,348 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T23:41:47,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T23:41:48,358 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-20T23:41:48,358 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-20T23:41:48,358 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:41:48,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-20T23:41:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2] 2024-11-20T23:41:49,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32785 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T23:41:50,186 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-20T23:41:50,357 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=412a5e44fd2e:32785 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=990ms, rpcTimeout=981ms 2024-11-20T23:41:50,358 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=990ms, rpcTimeout=981ms 2024-11-20T23:41:50,358 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., hostname=412a5e44fd2e,32785,1732146011021, seqNum=2 from cache 2024-11-20T23:41:50,366 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 234) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2227 (was 1318) - AvailableMemoryMB LEAK? - 2024-11-20T23:41:50,367 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T23:41:50,367 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T23:41:50,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T23:41:50,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T23:41:50,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T23:41:50,374 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T23:41:50,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T23:41:50,374 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T23:41:50,374 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-11-20T23:41:50,374 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=412a5e44fd2e,38443,1732146010136 2024-11-20T23:41:50,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T23:41:50,443 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T23:41:50,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:41:50,443 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:41:50,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T23:41:50,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T23:41:50,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T23:41:50,445 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T23:41:50,446 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T23:41:50,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,447 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '412a5e44fd2e,32785,1732146011021' ***** 2024-11-20T23:41:50,447 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T23:41:50,447 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T23:41:50,448 INFO [RS:0;412a5e44fd2e:32785 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T23:41:50,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T23:41:50,448 INFO [RS:0;412a5e44fd2e:32785 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T23:41:50,448 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 3830156c857c64aa567f2601eaf4755e 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 71c6fe5965cb00c238e2d6f3bcc565d1 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 69bf8e54f5b7d477f3a5e23f1012952d 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 428b56542c01920361d3b108ed2e6dd4 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 33a2ef971a60bc25e4ac3d3f03f5d530 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:50,449 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3830156c857c64aa567f2601eaf4755e, disabling compactions & flushes 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 0252bb9c4c7b2f32b61e2212a71858a1 2024-11-20T23:41:50,449 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:50,449 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(959): stopping server 412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:50,449 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. after waiting 0 ms 2024-11-20T23:41:50,449 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:50,449 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T23:41:50,450 INFO [RS:0;412a5e44fd2e:32785 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;412a5e44fd2e:32785. 2024-11-20T23:41:50,450 DEBUG [RS:0;412a5e44fd2e:32785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T23:41:50,450 DEBUG [RS:0;412a5e44fd2e:32785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:50,450 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T23:41:50,450 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T23:41:50,450 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T23:41:50,450 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T23:41:50,451 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-20T23:41:50,451 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T23:41:50,451 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T23:41:50,451 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 1588230740, 33a2ef971a60bc25e4ac3d3f03f5d530, 3830156c857c64aa567f2601eaf4755e, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a, 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:41:50,451 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T23:41:50,451 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T23:41:50,451 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T23:41:50,453 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3830156c857c64aa567f2601eaf4755e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:50,453 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-20T23:41:50,521 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/.tmp/testFamily/0885bf21d6034231a2426789e835a1f0 is 61, key is row-0/testFamily:testQualifier/1732146098243/Put/seqid=0 2024-11-20T23:41:50,529 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/info/18d74cf8b19f43fdbec4995df4ea045d is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0./info:regioninfo/1732146046557/Put/seqid=0 2024-11-20T23:41:50,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741856_1032 (size=20564) 2024-11-20T23:41:50,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741855_1031 (size=5049) 2024-11-20T23:41:50,547 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/.tmp/testFamily/0885bf21d6034231a2426789e835a1f0 2024-11-20T23:41:50,618 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/.tmp/testFamily/0885bf21d6034231a2426789e835a1f0 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/testFamily/0885bf21d6034231a2426789e835a1f0 2024-11-20T23:41:50,628 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/testFamily/0885bf21d6034231a2426789e835a1f0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:50,637 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3830156c857c64aa567f2601eaf4755e in 182ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:50,646 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNext/3830156c857c64aa567f2601eaf4755e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:50,648 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:50,648 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3830156c857c64aa567f2601eaf4755e: Waiting for close lock at 1732146110449Running coprocessor pre-close hooks at 1732146110449Disabling compacts and flushes for region at 1732146110449Disabling writes for close at 1732146110449Obtaining lock to block concurrent updates at 1732146110453 (+4 ms)Preparing flush snapshotting stores in 3830156c857c64aa567f2601eaf4755e at 1732146110453Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146110460 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. at 1732146110462 (+2 ms)Flushing 3830156c857c64aa567f2601eaf4755e/testFamily: creating writer at 1732146110464 (+2 ms)Flushing 3830156c857c64aa567f2601eaf4755e/testFamily: appending metadata at 1732146110513 (+49 ms)Flushing 3830156c857c64aa567f2601eaf4755e/testFamily: closing flushed file at 1732146110517 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fe909: reopening flushed file at 1732146110616 (+99 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3830156c857c64aa567f2601eaf4755e in 182ms, sequenceid=8, compaction requested=false at 1732146110637 (+21 ms)Writing region close event to WAL at 1732146110640 (+3 ms)Running coprocessor post-close hooks at 1732146110647 (+7 ms)Closed at 1732146110648 (+1 ms) 2024-11-20T23:41:50,649 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732146097216.3830156c857c64aa567f2601eaf4755e. 2024-11-20T23:41:50,649 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f341086aa6d427ae5e5e95a7a4ed0d0, disabling compactions & flushes 2024-11-20T23:41:50,649 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:41:50,649 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:41:50,649 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. after waiting 0 ms 2024-11-20T23:41:50,649 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:41:50,649 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9f341086aa6d427ae5e5e95a7a4ed0d0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:50,651 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 1588230740, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a, 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:41:50,673 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/.tmp/testFamily/195bef09fc8f4e88a6c7079b3d049fdd is 61, key is row-0/testFamily:testQualifier/1732146048382/Put/seqid=0 2024-11-20T23:41:50,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741857_1033 (size=5049) 2024-11-20T23:41:50,851 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 1588230740, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a, 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:41:50,870 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-20T23:41:50,942 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/info/18d74cf8b19f43fdbec4995df4ea045d 2024-11-20T23:41:50,943 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:50,989 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/ns/aae69bc59baa46b18b4de771201f6edc is 43, key is default/ns:d/1732146044807/Put/seqid=0 2024-11-20T23:41:50,989 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-20T23:41:50,989 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T23:41:50,990 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-20T23:41:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741858_1034 (size=5153) 2024-11-20T23:41:51,022 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/ns/aae69bc59baa46b18b4de771201f6edc 2024-11-20T23:41:51,052 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 1588230740, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a, 9f341086aa6d427ae5e5e95a7a4ed0d0 2024-11-20T23:41:51,078 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/table/4cdab82146674bdd9814977309034844 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732146046573/Put/seqid=0 2024-11-20T23:41:51,081 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/.tmp/testFamily/195bef09fc8f4e88a6c7079b3d049fdd 2024-11-20T23:41:51,085 INFO [regionserver/412a5e44fd2e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T23:41:51,085 INFO [regionserver/412a5e44fd2e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T23:41:51,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741859_1035 (size=6699) 2024-11-20T23:41:51,093 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/table/4cdab82146674bdd9814977309034844 2024-11-20T23:41:51,107 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/.tmp/testFamily/195bef09fc8f4e88a6c7079b3d049fdd as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily/195bef09fc8f4e88a6c7079b3d049fdd 2024-11-20T23:41:51,133 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily/195bef09fc8f4e88a6c7079b3d049fdd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:51,135 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f341086aa6d427ae5e5e95a7a4ed0d0 in 486ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:51,135 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/info/18d74cf8b19f43fdbec4995df4ea045d as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/info/18d74cf8b19f43fdbec4995df4ea045d 2024-11-20T23:41:51,151 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9f341086aa6d427ae5e5e95a7a4ed0d0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:51,152 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:41:51,153 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f341086aa6d427ae5e5e95a7a4ed0d0: Waiting for close lock at 1732146110649Running coprocessor pre-close hooks at 1732146110649Disabling compacts and flushes for region at 1732146110649Disabling writes for close at 1732146110649Obtaining lock to block concurrent updates at 1732146110649Preparing flush snapshotting stores in 9f341086aa6d427ae5e5e95a7a4ed0d0 at 1732146110649Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146110650 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. at 1732146110651 (+1 ms)Flushing 9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily: creating writer at 1732146110651Flushing 9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily: appending metadata at 1732146110672 (+21 ms)Flushing 9f341086aa6d427ae5e5e95a7a4ed0d0/testFamily: closing flushed file at 1732146110672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73b56806: reopening flushed file at 1732146111106 (+434 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f341086aa6d427ae5e5e95a7a4ed0d0 in 486ms, sequenceid=8, compaction requested=false at 1732146111135 (+29 ms)Writing region close event to WAL at 1732146111144 (+9 ms)Running coprocessor post-close hooks at 1732146111152 (+8 ms)Closed at 1732146111152 2024-11-20T23:41:51,153 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/info/18d74cf8b19f43fdbec4995df4ea045d, entries=100, sequenceid=56, filesize=20.1 K 2024-11-20T23:41:51,153 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732146045290.9f341086aa6d427ae5e5e95a7a4ed0d0. 2024-11-20T23:41:51,153 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 71c6fe5965cb00c238e2d6f3bcc565d1, disabling compactions & flushes 2024-11-20T23:41:51,153 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:51,153 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:51,153 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. after waiting 0 ms 2024-11-20T23:41:51,154 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:51,154 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 71c6fe5965cb00c238e2d6f3bcc565d1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:51,154 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/ns/aae69bc59baa46b18b4de771201f6edc as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/ns/aae69bc59baa46b18b4de771201f6edc 2024-11-20T23:41:51,175 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/ns/aae69bc59baa46b18b4de771201f6edc, entries=2, sequenceid=56, filesize=5.0 K 2024-11-20T23:41:51,182 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/.tmp/table/4cdab82146674bdd9814977309034844 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/table/4cdab82146674bdd9814977309034844 2024-11-20T23:41:51,182 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/.tmp/testFamily/75ad89e0b14f453d8f32d2bbb74688f7 is 61, key is row-0/testFamily:testQualifier/1732146085077/Put/seqid=0 2024-11-20T23:41:51,203 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/table/4cdab82146674bdd9814977309034844, entries=20, sequenceid=56, filesize=6.5 K 2024-11-20T23:41:51,205 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 754ms, sequenceid=56, compaction requested=false 2024-11-20T23:41:51,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741860_1036 (size=5049) 2024-11-20T23:41:51,225 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-20T23:41:51,227 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T23:41:51,227 INFO [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T23:41:51,228 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732146110451Running coprocessor pre-close hooks at 1732146110451Disabling compacts and flushes for region at 1732146110451Disabling writes for close at 1732146110451Obtaining lock to block concurrent updates at 1732146110453 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732146110453Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732146110460 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732146110462 (+2 ms)Flushing 1588230740/info: creating writer at 1732146110464 (+2 ms)Flushing 1588230740/info: appending metadata at 1732146110522 (+58 ms)Flushing 1588230740/info: closing flushed file at 1732146110522Flushing 1588230740/ns: creating writer at 1732146110964 (+442 ms)Flushing 1588230740/ns: appending metadata at 1732146110987 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1732146110987Flushing 1588230740/table: creating writer at 1732146111039 (+52 ms)Flushing 1588230740/table: appending metadata at 1732146111077 (+38 ms)Flushing 1588230740/table: closing flushed file at 1732146111077Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4260455: reopening flushed file at 1732146111131 (+54 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@511484fe: reopening flushed file at 1732146111153 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28800f9c: reopening flushed file at 1732146111175 (+22 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 754ms, sequenceid=56, compaction requested=false at 1732146111205 (+30 ms)Writing region close event to WAL at 1732146111216 (+11 ms)Running coprocessor post-close hooks at 1732146111227 (+11 ms)Closed at 1732146111227 2024-11-20T23:41:51,228 DEBUG [RS_CLOSE_META-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T23:41:51,252 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:51,453 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-11-20T23:41:51,453 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1325): Online Regions={71c6fe5965cb00c238e2d6f3bcc565d1=testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1., 69bf8e54f5b7d477f3a5e23f1012952d=testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., 96847a8f1dc149e4522cad8ea9883e5a=hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a., 428b56542c01920361d3b108ed2e6dd4=testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4., 33a2ef971a60bc25e4ac3d3f03f5d530=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., 823bc8b5555cc1423e230ccf63c5f45a=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., 0252bb9c4c7b2f32b61e2212a71858a1=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., 7ee1609c7c5d894dcc55416f0d9960a5=testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.} 2024-11-20T23:41:51,453 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:51,615 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/.tmp/testFamily/75ad89e0b14f453d8f32d2bbb74688f7 2024-11-20T23:41:51,632 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/.tmp/testFamily/75ad89e0b14f453d8f32d2bbb74688f7 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/testFamily/75ad89e0b14f453d8f32d2bbb74688f7 2024-11-20T23:41:51,650 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/testFamily/75ad89e0b14f453d8f32d2bbb74688f7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:51,652 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 71c6fe5965cb00c238e2d6f3bcc565d1 in 498ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:51,653 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 71c6fe5965cb00c238e2d6f3bcc565d1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:51,677 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNext/71c6fe5965cb00c238e2d6f3bcc565d1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:51,679 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:51,680 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 71c6fe5965cb00c238e2d6f3bcc565d1: Waiting for close lock at 1732146111153Running coprocessor pre-close hooks at 1732146111153Disabling compacts and flushes for region at 1732146111153Disabling writes for close at 1732146111154 (+1 ms)Obtaining lock to block concurrent updates at 1732146111154Preparing flush snapshotting stores in 71c6fe5965cb00c238e2d6f3bcc565d1 at 1732146111154Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146111154Flushing stores of testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. at 1732146111155 (+1 ms)Flushing 71c6fe5965cb00c238e2d6f3bcc565d1/testFamily: creating writer at 1732146111156 (+1 ms)Flushing 71c6fe5965cb00c238e2d6f3bcc565d1/testFamily: appending metadata at 1732146111182 (+26 ms)Flushing 71c6fe5965cb00c238e2d6f3bcc565d1/testFamily: closing flushed file at 1732146111182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50ec399e: reopening flushed file at 1732146111628 (+446 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 71c6fe5965cb00c238e2d6f3bcc565d1 in 498ms, sequenceid=8, compaction requested=false at 1732146111652 (+24 ms)Writing region close event to WAL at 1732146111664 (+12 ms)Running coprocessor post-close hooks at 1732146111678 (+14 ms)Closed at 1732146111679 (+1 ms) 2024-11-20T23:41:51,680 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732146082035.71c6fe5965cb00c238e2d6f3bcc565d1. 2024-11-20T23:41:51,681 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69bf8e54f5b7d477f3a5e23f1012952d, disabling compactions & flushes 2024-11-20T23:41:51,681 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:51,681 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:51,681 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. after waiting 0 ms 2024-11-20T23:41:51,681 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:51,681 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 69bf8e54f5b7d477f3a5e23f1012952d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:51,712 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/.tmp/testFamily/d5c9555676e64fb2a573408f797fc603 is 61, key is row-0/testFamily:testQualifier/1732146072911/Put/seqid=0 2024-11-20T23:41:51,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741861_1037 (size=5049) 2024-11-20T23:41:51,854 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:52,054 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 69bf8e54f5b7d477f3a5e23f1012952d, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:52,119 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/.tmp/testFamily/d5c9555676e64fb2a573408f797fc603 2024-11-20T23:41:52,130 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/.tmp/testFamily/d5c9555676e64fb2a573408f797fc603 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/testFamily/d5c9555676e64fb2a573408f797fc603 2024-11-20T23:41:52,149 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/testFamily/d5c9555676e64fb2a573408f797fc603, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:52,151 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69bf8e54f5b7d477f3a5e23f1012952d in 470ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:52,165 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScanner/69bf8e54f5b7d477f3a5e23f1012952d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:52,166 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:52,166 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69bf8e54f5b7d477f3a5e23f1012952d: Waiting for close lock at 1732146111680Running coprocessor pre-close hooks at 1732146111681 (+1 ms)Disabling compacts and flushes for region at 1732146111681Disabling writes for close at 1732146111681Obtaining lock to block concurrent updates at 1732146111681Preparing flush snapshotting stores in 69bf8e54f5b7d477f3a5e23f1012952d at 1732146111681Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146111681Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. at 1732146111683 (+2 ms)Flushing 69bf8e54f5b7d477f3a5e23f1012952d/testFamily: creating writer at 1732146111683Flushing 69bf8e54f5b7d477f3a5e23f1012952d/testFamily: appending metadata at 1732146111711 (+28 ms)Flushing 69bf8e54f5b7d477f3a5e23f1012952d/testFamily: closing flushed file at 1732146111711Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a3117c3: reopening flushed file at 1732146112129 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 69bf8e54f5b7d477f3a5e23f1012952d in 470ms, sequenceid=8, compaction requested=false at 1732146112151 (+22 ms)Writing region close event to WAL at 1732146112155 (+4 ms)Running coprocessor post-close hooks at 1732146112166 (+11 ms)Closed at 1732146112166 2024-11-20T23:41:52,167 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732146069866.69bf8e54f5b7d477f3a5e23f1012952d. 2024-11-20T23:41:52,167 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96847a8f1dc149e4522cad8ea9883e5a, disabling compactions & flushes 2024-11-20T23:41:52,167 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:41:52,167 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:41:52,167 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. after waiting 0 ms 2024-11-20T23:41:52,167 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:41:52,167 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 96847a8f1dc149e4522cad8ea9883e5a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:52,189 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/.tmp/testFamily/06fafc03df30495c9ed8e96f460ebba6 is 61, key is row-0/testFamily:testQualifier/1732146060683/Put/seqid=0 2024-11-20T23:41:52,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741862_1038 (size=5049) 2024-11-20T23:41:52,254 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:52,454 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-20T23:41:52,455 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1325): Online Regions={96847a8f1dc149e4522cad8ea9883e5a=hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a., 428b56542c01920361d3b108ed2e6dd4=testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4., 33a2ef971a60bc25e4ac3d3f03f5d530=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., 823bc8b5555cc1423e230ccf63c5f45a=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., 0252bb9c4c7b2f32b61e2212a71858a1=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., 7ee1609c7c5d894dcc55416f0d9960a5=testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.} 2024-11-20T23:41:52,455 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a, 96847a8f1dc149e4522cad8ea9883e5a 2024-11-20T23:41:52,601 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/.tmp/testFamily/06fafc03df30495c9ed8e96f460ebba6 2024-11-20T23:41:52,610 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/.tmp/testFamily/06fafc03df30495c9ed8e96f460ebba6 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/testFamily/06fafc03df30495c9ed8e96f460ebba6 2024-11-20T23:41:52,618 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/testFamily/06fafc03df30495c9ed8e96f460ebba6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:52,619 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96847a8f1dc149e4522cad8ea9883e5a in 452ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:52,624 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnNextAsync/96847a8f1dc149e4522cad8ea9883e5a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:52,625 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:41:52,625 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96847a8f1dc149e4522cad8ea9883e5a: Waiting for close lock at 1732146112167Running coprocessor pre-close hooks at 1732146112167Disabling compacts and flushes for region at 1732146112167Disabling writes for close at 1732146112167Obtaining lock to block concurrent updates at 1732146112167Preparing flush snapshotting stores in 96847a8f1dc149e4522cad8ea9883e5a at 1732146112167Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146112167Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. at 1732146112170 (+3 ms)Flushing 96847a8f1dc149e4522cad8ea9883e5a/testFamily: creating writer at 1732146112171 (+1 ms)Flushing 96847a8f1dc149e4522cad8ea9883e5a/testFamily: appending metadata at 1732146112188 (+17 ms)Flushing 96847a8f1dc149e4522cad8ea9883e5a/testFamily: closing flushed file at 1732146112188Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30619921: reopening flushed file at 1732146112609 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 96847a8f1dc149e4522cad8ea9883e5a in 452ms, sequenceid=8, compaction requested=false at 1732146112619 (+10 ms)Writing region close event to WAL at 1732146112620 (+1 ms)Running coprocessor post-close hooks at 1732146112625 (+5 ms)Closed at 1732146112625 2024-11-20T23:41:52,626 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732146057639.96847a8f1dc149e4522cad8ea9883e5a. 2024-11-20T23:41:52,626 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 428b56542c01920361d3b108ed2e6dd4, disabling compactions & flushes 2024-11-20T23:41:52,626 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:52,626 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:52,626 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. after waiting 0 ms 2024-11-20T23:41:52,626 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:52,626 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 428b56542c01920361d3b108ed2e6dd4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:52,648 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/.tmp/testFamily/4d4d929018254edf9788fd2f88f21028 is 61, key is row-0/testFamily:testQualifier/1732146078995/Put/seqid=0 2024-11-20T23:41:52,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741863_1039 (size=5049) 2024-11-20T23:41:52,655 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:52,855 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:52,891 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=412a5e44fd2e:38443 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 10 more 2024-11-20T23:41:53,055 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/.tmp/testFamily/4d4d929018254edf9788fd2f88f21028 2024-11-20T23:41:53,056 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 428b56542c01920361d3b108ed2e6dd4, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:53,069 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/.tmp/testFamily/4d4d929018254edf9788fd2f88f21028 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/testFamily/4d4d929018254edf9788fd2f88f21028 2024-11-20T23:41:53,078 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/testFamily/4d4d929018254edf9788fd2f88f21028, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:53,080 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 428b56542c01920361d3b108ed2e6dd4 in 454ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:53,085 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnNextAsync/428b56542c01920361d3b108ed2e6dd4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:53,086 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:53,086 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 428b56542c01920361d3b108ed2e6dd4: Waiting for close lock at 1732146112626Running coprocessor pre-close hooks at 1732146112626Disabling compacts and flushes for region at 1732146112626Disabling writes for close at 1732146112626Obtaining lock to block concurrent updates at 1732146112626Preparing flush snapshotting stores in 428b56542c01920361d3b108ed2e6dd4 at 1732146112626Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146112627 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. at 1732146112627Flushing 428b56542c01920361d3b108ed2e6dd4/testFamily: creating writer at 1732146112627Flushing 428b56542c01920361d3b108ed2e6dd4/testFamily: appending metadata at 1732146112648 (+21 ms)Flushing 428b56542c01920361d3b108ed2e6dd4/testFamily: closing flushed file at 1732146112648Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c1a9ad: reopening flushed file at 1732146113068 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 428b56542c01920361d3b108ed2e6dd4 in 454ms, sequenceid=8, compaction requested=false at 1732146113080 (+12 ms)Writing region close event to WAL at 1732146113081 (+1 ms)Running coprocessor post-close hooks at 1732146113086 (+5 ms)Closed at 1732146113086 2024-11-20T23:41:53,086 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732146075949.428b56542c01920361d3b108ed2e6dd4. 2024-11-20T23:41:53,086 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33a2ef971a60bc25e4ac3d3f03f5d530, disabling compactions & flushes 2024-11-20T23:41:53,087 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:53,087 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:53,087 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. after waiting 0 ms 2024-11-20T23:41:53,087 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:53,087 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 33a2ef971a60bc25e4ac3d3f03f5d530 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:53,104 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/.tmp/testFamily/1b854457e7004f82b9a80d9414a8b5cd is 61, key is row-0/testFamily:testQualifier/1732146107334/Put/seqid=0 2024-11-20T23:41:53,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741864_1040 (size=5049) 2024-11-20T23:41:53,256 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:53,456 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-20T23:41:53,456 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1325): Online Regions={33a2ef971a60bc25e4ac3d3f03f5d530=testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., 823bc8b5555cc1423e230ccf63c5f45a=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., 0252bb9c4c7b2f32b61e2212a71858a1=hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., 7ee1609c7c5d894dcc55416f0d9960a5=testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.} 2024-11-20T23:41:53,456 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 33a2ef971a60bc25e4ac3d3f03f5d530, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:53,510 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/.tmp/testFamily/1b854457e7004f82b9a80d9414a8b5cd 2024-11-20T23:41:53,519 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/.tmp/testFamily/1b854457e7004f82b9a80d9414a8b5cd as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/testFamily/1b854457e7004f82b9a80d9414a8b5cd 2024-11-20T23:41:53,528 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/testFamily/1b854457e7004f82b9a80d9414a8b5cd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:53,530 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 33a2ef971a60bc25e4ac3d3f03f5d530 in 443ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:53,535 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testNormalScanTimeoutOnOpenScannerAsync/33a2ef971a60bc25e4ac3d3f03f5d530/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:53,536 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:53,536 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33a2ef971a60bc25e4ac3d3f03f5d530: Waiting for close lock at 1732146113086Running coprocessor pre-close hooks at 1732146113086Disabling compacts and flushes for region at 1732146113086Disabling writes for close at 1732146113087 (+1 ms)Obtaining lock to block concurrent updates at 1732146113087Preparing flush snapshotting stores in 33a2ef971a60bc25e4ac3d3f03f5d530 at 1732146113087Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146113087Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. at 1732146113088 (+1 ms)Flushing 33a2ef971a60bc25e4ac3d3f03f5d530/testFamily: creating writer at 1732146113088Flushing 33a2ef971a60bc25e4ac3d3f03f5d530/testFamily: appending metadata at 1732146113104 (+16 ms)Flushing 33a2ef971a60bc25e4ac3d3f03f5d530/testFamily: closing flushed file at 1732146113104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d65ad2d: reopening flushed file at 1732146113518 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 33a2ef971a60bc25e4ac3d3f03f5d530 in 443ms, sequenceid=8, compaction requested=false at 1732146113530 (+12 ms)Writing region close event to WAL at 1732146113530Running coprocessor post-close hooks at 1732146113536 (+6 ms)Closed at 1732146113536 2024-11-20T23:41:53,536 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732146104284.33a2ef971a60bc25e4ac3d3f03f5d530. 2024-11-20T23:41:53,537 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 823bc8b5555cc1423e230ccf63c5f45a, disabling compactions & flushes 2024-11-20T23:41:53,537 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:53,537 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:53,537 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. after waiting 0 ms 2024-11-20T23:41:53,537 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:53,537 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 823bc8b5555cc1423e230ccf63c5f45a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:53,561 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/.tmp/testFamily/7985ad70f8414a94a276c3ed4b5f001b is 61, key is row-0/testFamily:testQualifier/1732146091176/Put/seqid=0 2024-11-20T23:41:53,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741865_1041 (size=5049) 2024-11-20T23:41:53,657 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:53,857 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 7ee1609c7c5d894dcc55416f0d9960a5, 823bc8b5555cc1423e230ccf63c5f45a 2024-11-20T23:41:53,971 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/.tmp/testFamily/7985ad70f8414a94a276c3ed4b5f001b 2024-11-20T23:41:53,985 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/.tmp/testFamily/7985ad70f8414a94a276c3ed4b5f001b as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/testFamily/7985ad70f8414a94a276c3ed4b5f001b 2024-11-20T23:41:53,995 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/testFamily/7985ad70f8414a94a276c3ed4b5f001b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:53,996 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 823bc8b5555cc1423e230ccf63c5f45a in 459ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:54,003 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/823bc8b5555cc1423e230ccf63c5f45a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:54,003 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 823bc8b5555cc1423e230ccf63c5f45a: Waiting for close lock at 1732146113537Running coprocessor pre-close hooks at 1732146113537Disabling compacts and flushes for region at 1732146113537Disabling writes for close at 1732146113537Obtaining lock to block concurrent updates at 1732146113537Preparing flush snapshotting stores in 823bc8b5555cc1423e230ccf63c5f45a at 1732146113537Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146113537Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. at 1732146113538 (+1 ms)Flushing 823bc8b5555cc1423e230ccf63c5f45a/testFamily: creating writer at 1732146113538Flushing 823bc8b5555cc1423e230ccf63c5f45a/testFamily: appending metadata at 1732146113561 (+23 ms)Flushing 823bc8b5555cc1423e230ccf63c5f45a/testFamily: closing flushed file at 1732146113561Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26239967: reopening flushed file at 1732146113983 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 823bc8b5555cc1423e230ccf63c5f45a in 459ms, sequenceid=8, compaction requested=false at 1732146113996 (+13 ms)Writing region close event to WAL at 1732146113999 (+3 ms)Running coprocessor post-close hooks at 1732146114003 (+4 ms)Closed at 1732146114003 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732146088132.823bc8b5555cc1423e230ccf63c5f45a. 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0252bb9c4c7b2f32b61e2212a71858a1, disabling compactions & flushes 2024-11-20T23:41:54,004 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. after waiting 0 ms 2024-11-20T23:41:54,004 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:41:54,004 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0252bb9c4c7b2f32b61e2212a71858a1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:54,021 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/.tmp/testFamily/7fd0d306e6c349b0b4050e968b8347f9 is 61, key is row-0/testFamily:testQualifier/1732146051546/Put/seqid=0 2024-11-20T23:41:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741866_1042 (size=5049) 2024-11-20T23:41:54,057 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:54,257 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 0252bb9c4c7b2f32b61e2212a71858a1, 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:54,426 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/.tmp/testFamily/7fd0d306e6c349b0b4050e968b8347f9 2024-11-20T23:41:54,437 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/.tmp/testFamily/7fd0d306e6c349b0b4050e968b8347f9 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/testFamily/7fd0d306e6c349b0b4050e968b8347f9 2024-11-20T23:41:54,447 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/testFamily/7fd0d306e6c349b0b4050e968b8347f9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:54,448 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0252bb9c4c7b2f32b61e2212a71858a1 in 444ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:54,454 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/hbase/testMetaScanTimeoutOnOpenScanner/0252bb9c4c7b2f32b61e2212a71858a1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:54,455 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:41:54,455 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0252bb9c4c7b2f32b61e2212a71858a1: Waiting for close lock at 1732146114004Running coprocessor pre-close hooks at 1732146114004Disabling compacts and flushes for region at 1732146114004Disabling writes for close at 1732146114004Obtaining lock to block concurrent updates at 1732146114004Preparing flush snapshotting stores in 0252bb9c4c7b2f32b61e2212a71858a1 at 1732146114004Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146114004Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. at 1732146114005 (+1 ms)Flushing 0252bb9c4c7b2f32b61e2212a71858a1/testFamily: creating writer at 1732146114005Flushing 0252bb9c4c7b2f32b61e2212a71858a1/testFamily: appending metadata at 1732146114021 (+16 ms)Flushing 0252bb9c4c7b2f32b61e2212a71858a1/testFamily: closing flushed file at 1732146114021Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3375197c: reopening flushed file at 1732146114435 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0252bb9c4c7b2f32b61e2212a71858a1 in 444ms, sequenceid=8, compaction requested=false at 1732146114448 (+13 ms)Writing region close event to WAL at 1732146114449 (+1 ms)Running coprocessor post-close hooks at 1732146114455 (+6 ms)Closed at 1732146114455 2024-11-20T23:41:54,456 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732146048479.0252bb9c4c7b2f32b61e2212a71858a1. 2024-11-20T23:41:54,456 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ee1609c7c5d894dcc55416f0d9960a5, disabling compactions & flushes 2024-11-20T23:41:54,456 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:54,456 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:54,456 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. after waiting 0 ms 2024-11-20T23:41:54,456 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:54,456 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7ee1609c7c5d894dcc55416f0d9960a5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T23:41:54,458 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T23:41:54,458 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1325): Online Regions={7ee1609c7c5d894dcc55416f0d9960a5=testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5.} 2024-11-20T23:41:54,458 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:54,481 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/.tmp/testFamily/c74a4a7348a14f6094943d71ed361acb is 61, key is row-0/testFamily:testQualifier/1732146069788/Put/seqid=0 2024-11-20T23:41:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741867_1043 (size=5049) 2024-11-20T23:41:54,658 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:54,858 DEBUG [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1351): Waiting on 7ee1609c7c5d894dcc55416f0d9960a5 2024-11-20T23:41:54,886 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/.tmp/testFamily/c74a4a7348a14f6094943d71ed361acb 2024-11-20T23:41:54,895 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/.tmp/testFamily/c74a4a7348a14f6094943d71ed361acb as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/testFamily/c74a4a7348a14f6094943d71ed361acb 2024-11-20T23:41:54,903 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/testFamily/c74a4a7348a14f6094943d71ed361acb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T23:41:54,905 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ee1609c7c5d894dcc55416f0d9960a5 in 449ms, sequenceid=8, compaction requested=false 2024-11-20T23:41:54,910 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/data/default/testRetryOutOfOrderScannerNextException/7ee1609c7c5d894dcc55416f0d9960a5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T23:41:54,911 INFO [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:54,911 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ee1609c7c5d894dcc55416f0d9960a5: Waiting for close lock at 1732146114456Running coprocessor pre-close hooks at 1732146114456Disabling compacts and flushes for region at 1732146114456Disabling writes for close at 1732146114456Obtaining lock to block concurrent updates at 1732146114456Preparing flush snapshotting stores in 7ee1609c7c5d894dcc55416f0d9960a5 at 1732146114456Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732146114457 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. at 1732146114457Flushing 7ee1609c7c5d894dcc55416f0d9960a5/testFamily: creating writer at 1732146114457Flushing 7ee1609c7c5d894dcc55416f0d9960a5/testFamily: appending metadata at 1732146114480 (+23 ms)Flushing 7ee1609c7c5d894dcc55416f0d9960a5/testFamily: closing flushed file at 1732146114480Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65495b7a: reopening flushed file at 1732146114894 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ee1609c7c5d894dcc55416f0d9960a5 in 449ms, sequenceid=8, compaction requested=false at 1732146114905 (+11 ms)Writing region close event to WAL at 1732146114906 (+1 ms)Running coprocessor post-close hooks at 1732146114911 (+5 ms)Closed at 1732146114911 2024-11-20T23:41:54,911 DEBUG [RS_CLOSE_REGION-regionserver/412a5e44fd2e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732146066729.7ee1609c7c5d894dcc55416f0d9960a5. 2024-11-20T23:41:55,059 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(976): stopping server 412a5e44fd2e,32785,1732146011021; all regions closed. 2024-11-20T23:41:55,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741834_1010 (size=19479) 2024-11-20T23:41:55,068 DEBUG [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/oldWALs 2024-11-20T23:41:55,068 INFO [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 412a5e44fd2e%2C32785%2C1732146011021.meta:.meta(num 1732146044507) 2024-11-20T23:41:55,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741833_1009 (size=20031) 2024-11-20T23:41:55,070 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/WALs/412a5e44fd2e,32785,1732146011021/412a5e44fd2e%2C32785%2C1732146011021.1732146044077 not finished, retry = 0 2024-11-20T23:41:55,177 DEBUG [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/oldWALs 2024-11-20T23:41:55,177 INFO [RS:0;412a5e44fd2e:32785 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 412a5e44fd2e%2C32785%2C1732146011021:(num 1732146044077) 2024-11-20T23:41:55,177 DEBUG [RS:0;412a5e44fd2e:32785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T23:41:55,178 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T23:41:55,178 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T23:41:55,178 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.ChoreService(370): Chore service for: regionserver/412a5e44fd2e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T23:41:55,179 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T23:41:55,179 INFO [regionserver/412a5e44fd2e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T23:41:55,182 INFO [RS:0;412a5e44fd2e:32785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32785 2024-11-20T23:41:55,240 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/412a5e44fd2e,32785,1732146011021 2024-11-20T23:41:55,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T23:41:55,240 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T23:41:55,243 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [412a5e44fd2e,32785,1732146011021] 2024-11-20T23:41:55,248 INFO [regionserver/412a5e44fd2e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T23:41:55,277 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/412a5e44fd2e,32785,1732146011021 already deleted, retry=false 2024-11-20T23:41:55,277 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 412a5e44fd2e,32785,1732146011021 expired; onlineServers=0 2024-11-20T23:41:55,277 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '412a5e44fd2e,38443,1732146010136' ***** 2024-11-20T23:41:55,278 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T23:41:55,278 INFO [M:0;412a5e44fd2e:38443 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T23:41:55,278 INFO [M:0;412a5e44fd2e:38443 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T23:41:55,278 DEBUG [M:0;412a5e44fd2e:38443 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T23:41:55,278 DEBUG [M:0;412a5e44fd2e:38443 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T23:41:55,278 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.large.0-1732146013886 {}] cleaner.HFileCleaner(306): Exit Thread[master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.large.0-1732146013886,5,FailOnTimeoutGroup] 2024-11-20T23:41:55,279 DEBUG [master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.small.0-1732146013887 {}] cleaner.HFileCleaner(306): Exit Thread[master/412a5e44fd2e:0:becomeActiveMaster-HFileCleaner.small.0-1732146013887,5,FailOnTimeoutGroup] 2024-11-20T23:41:55,279 INFO [M:0;412a5e44fd2e:38443 {}] hbase.ChoreService(370): Chore service for: master/412a5e44fd2e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T23:41:55,279 INFO [M:0;412a5e44fd2e:38443 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T23:41:55,279 DEBUG [M:0;412a5e44fd2e:38443 {}] master.HMaster(1795): Stopping service threads 2024-11-20T23:41:55,279 INFO [M:0;412a5e44fd2e:38443 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T23:41:55,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T23:41:55,279 INFO [M:0;412a5e44fd2e:38443 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T23:41:55,280 INFO [M:0;412a5e44fd2e:38443 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T23:41:55,281 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T23:41:55,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T23:41:55,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T23:41:55,288 DEBUG [M:0;412a5e44fd2e:38443 {}] zookeeper.ZKUtil(347): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T23:41:55,288 WARN [M:0;412a5e44fd2e:38443 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T23:41:55,296 INFO [M:0;412a5e44fd2e:38443 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/.lastflushedseqids 2024-11-20T23:41:55,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741868_1044 (size=649) 2024-11-20T23:41:55,367 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T23:41:55,367 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32785-0x1015a9e90aa0001, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T23:41:55,368 INFO [RS:0;412a5e44fd2e:32785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T23:41:55,368 INFO [RS:0;412a5e44fd2e:32785 {}] regionserver.HRegionServer(1031): Exiting; stopping=412a5e44fd2e,32785,1732146011021; zookeeper connection closed. 2024-11-20T23:41:55,368 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c6e23ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c6e23ac 2024-11-20T23:41:55,369 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T23:41:55,736 INFO [M:0;412a5e44fd2e:38443 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T23:41:55,736 INFO [M:0;412a5e44fd2e:38443 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T23:41:55,736 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T23:41:55,736 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:41:55,737 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:41:55,737 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T23:41:55,737 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:41:55,737 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.98 KB heapSize=190.87 KB 2024-11-20T23:41:55,759 DEBUG [M:0;412a5e44fd2e:38443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1435440b025a40229432ea174f9bb99e is 82, key is hbase:meta,,1/info:regioninfo/1732146044644/Put/seqid=0 2024-11-20T23:41:55,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741869_1045 (size=5672) 2024-11-20T23:41:56,150 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T23:41:56,167 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1435440b025a40229432ea174f9bb99e 2024-11-20T23:41:56,212 DEBUG [M:0;412a5e44fd2e:38443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45c64c9825f040e58aace67947d31b32 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732146046581/Put/seqid=0 2024-11-20T23:41:56,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741870_1046 (size=14069) 2024-11-20T23:41:56,221 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.42 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45c64c9825f040e58aace67947d31b32 2024-11-20T23:41:56,232 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45c64c9825f040e58aace67947d31b32 2024-11-20T23:41:56,264 DEBUG [M:0;412a5e44fd2e:38443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/730a7edac38b4c86b46896c63d3dc984 is 69, key is 412a5e44fd2e,32785,1732146011021/rs:state/1732146013943/Put/seqid=0 2024-11-20T23:41:56,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741871_1047 (size=5156) 2024-11-20T23:41:56,693 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/730a7edac38b4c86b46896c63d3dc984 2024-11-20T23:41:56,700 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1435440b025a40229432ea174f9bb99e as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1435440b025a40229432ea174f9bb99e 2024-11-20T23:41:56,708 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1435440b025a40229432ea174f9bb99e, entries=8, sequenceid=317, filesize=5.5 K 2024-11-20T23:41:56,709 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45c64c9825f040e58aace67947d31b32 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45c64c9825f040e58aace67947d31b32 2024-11-20T23:41:56,716 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 45c64c9825f040e58aace67947d31b32 2024-11-20T23:41:56,716 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45c64c9825f040e58aace67947d31b32, entries=33, sequenceid=317, filesize=13.7 K 2024-11-20T23:41:56,718 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/730a7edac38b4c86b46896c63d3dc984 as hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/730a7edac38b4c86b46896c63d3dc984 2024-11-20T23:41:56,724 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35501/user/jenkins/test-data/725ec830-e23a-d42b-c975-d7f872f6d37f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/730a7edac38b4c86b46896c63d3dc984, entries=1, sequenceid=317, filesize=5.0 K 2024-11-20T23:41:56,725 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 988ms, sequenceid=317, compaction requested=false 2024-11-20T23:41:56,727 INFO [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T23:41:56,727 DEBUG [M:0;412a5e44fd2e:38443 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732146115736Disabling compacts and flushes for region at 1732146115736Disabling writes for close at 1732146115737 (+1 ms)Obtaining lock to block concurrent updates at 1732146115737Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732146115737Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166891, getHeapSize=195384, getOffHeapSize=0, getCellsCount=364 at 1732146115737Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732146115738 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732146115738Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732146115759 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732146115759Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732146116187 (+428 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732146116211 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732146116211Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732146116232 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732146116263 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732146116263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e214256: reopening flushed file at 1732146116699 (+436 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622d7013: reopening flushed file at 1732146116708 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d95d9c0: reopening flushed file at 1732146116717 (+9 ms)Finished flush of dataSize ~162.98 KB/166891, heapSize ~190.57 KB/195144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 988ms, sequenceid=317, compaction requested=false at 1732146116725 (+8 ms)Writing region close event to WAL at 1732146116727 (+2 ms)Closed at 1732146116727 2024-11-20T23:41:56,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35447 is added to blk_1073741830_1006 (size=188103) 2024-11-20T23:41:56,731 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T23:41:56,731 INFO [M:0;412a5e44fd2e:38443 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T23:41:56,731 INFO [M:0;412a5e44fd2e:38443 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38443 2024-11-20T23:41:56,732 INFO [M:0;412a5e44fd2e:38443 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T23:41:56,840 INFO [M:0;412a5e44fd2e:38443 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T23:41:56,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T23:41:56,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38443-0x1015a9e90aa0000, quorum=127.0.0.1:53297, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T23:41:56,851 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc78d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T23:41:56,854 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T23:41:56,854 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T23:41:56,854 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T23:41:56,854 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.log.dir/,STOPPED} 2024-11-20T23:41:56,857 WARN [BP-2020952472-172.17.0.2-1732146006291 heartbeating to localhost/127.0.0.1:35501 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T23:41:56,857 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T23:41:56,857 WARN [BP-2020952472-172.17.0.2-1732146006291 heartbeating to localhost/127.0.0.1:35501 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2020952472-172.17.0.2-1732146006291 (Datanode Uuid 942ccc57-68d5-4f40-972f-449228f06d9f) service to localhost/127.0.0.1:35501 2024-11-20T23:41:56,857 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T23:41:56,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/data/data1/current/BP-2020952472-172.17.0.2-1732146006291 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T23:41:56,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/cluster_2c69bf3d-67da-70a5-7fe2-b5f0554f92c7/data/data2/current/BP-2020952472-172.17.0.2-1732146006291 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T23:41:56,860 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T23:41:56,876 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T23:41:56,877 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T23:41:56,877 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T23:41:56,877 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T23:41:56,877 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ecca1941-8635-70d9-9710-c5d58c2986f5/hadoop.log.dir/,STOPPED} 2024-11-20T23:41:56,887 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T23:41:56,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down