2024-11-20 02:56:11,545 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 02:56:11,562 main DEBUG Took 0.009692 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 02:56:11,562 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 02:56:11,562 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 02:56:11,563 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 02:56:11,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,570 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 02:56:11,581 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,583 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,583 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,585 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,585 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,586 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,587 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,588 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,589 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,591 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:56:11,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,593 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 02:56:11,594 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:56:11,595 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 02:56:11,597 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 02:56:11,597 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 02:56:11,599 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 02:56:11,599 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 02:56:11,606 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 02:56:11,608 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 02:56:11,610 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 02:56:11,610 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 02:56:11,611 main DEBUG createAppenders(={Console}) 2024-11-20 02:56:11,612 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-20 02:56:11,612 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 02:56:11,612 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-20 02:56:11,613 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 02:56:11,613 main DEBUG OutputStream closed 2024-11-20 02:56:11,613 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 02:56:11,613 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 02:56:11,614 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-20 02:56:11,692 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 02:56:11,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 02:56:11,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 02:56:11,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 02:56:11,696 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 02:56:11,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 02:56:11,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 02:56:11,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 02:56:11,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 02:56:11,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 02:56:11,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 02:56:11,698 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 02:56:11,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 02:56:11,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 02:56:11,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 02:56:11,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 02:56:11,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 02:56:11,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 02:56:11,703 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 02:56:11,703 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-20 02:56:11,703 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 02:56:11,704 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-20T02:56:11,715 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-20 02:56:11,718 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 02:56:11,718 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T02:56:11,930 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4 2024-11-20T02:56:11,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T02:56:11,968 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608, deleteOnExit=true 2024-11-20T02:56:11,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T02:56:11,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/test.cache.data in system properties and HBase conf 2024-11-20T02:56:11,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T02:56:11,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.log.dir in system properties and HBase conf 2024-11-20T02:56:11,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T02:56:11,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T02:56:11,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T02:56:12,067 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T02:56:12,170 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:56:12,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T02:56:12,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T02:56:12,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T02:56:12,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T02:56:12,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T02:56:12,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T02:56:12,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T02:56:12,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T02:56:12,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T02:56:12,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/nfs.dump.dir in system properties and HBase conf 2024-11-20T02:56:12,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/java.io.tmpdir in system properties and HBase conf 2024-11-20T02:56:12,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T02:56:12,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T02:56:12,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T02:56:13,229 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T02:56:13,298 INFO [Time-limited test {}] log.Log(170): Logging initialized @2428ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T02:56:13,364 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T02:56:13,421 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T02:56:13,441 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T02:56:13,442 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T02:56:13,443 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T02:56:13,454 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T02:56:13,457 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.log.dir/,AVAILABLE} 2024-11-20T02:56:13,458 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T02:56:13,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/java.io.tmpdir/jetty-localhost-43027-hadoop-hdfs-3_4_1-tests_jar-_-any-6582698994131684225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T02:56:13,648 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:43027} 2024-11-20T02:56:13,649 INFO [Time-limited test {}] server.Server(415): Started @2779ms 2024-11-20T02:56:14,146 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T02:56:14,153 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T02:56:14,155 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T02:56:14,155 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T02:56:14,155 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T02:56:14,156 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.log.dir/,AVAILABLE} 2024-11-20T02:56:14,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T02:56:14,257 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/java.io.tmpdir/jetty-localhost-39649-hadoop-hdfs-3_4_1-tests_jar-_-any-12299962459193834467/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T02:56:14,257 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:39649} 2024-11-20T02:56:14,258 INFO [Time-limited test {}] server.Server(415): Started @3388ms 2024-11-20T02:56:14,305 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T02:56:15,161 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/data/data1/current/BP-2052504977-172.17.0.2-1732071372787/current, will proceed with Du for space computation calculation, 2024-11-20T02:56:15,161 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/data/data2/current/BP-2052504977-172.17.0.2-1732071372787/current, will proceed with Du for space computation calculation, 2024-11-20T02:56:15,185 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T02:56:15,227 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x830245ccd7fbb08b with lease ID 0x9ff821ef1d811f02: Processing first storage report for DS-0472fe11-1b2f-4ca9-997f-b1908e24237b from datanode DatanodeRegistration(127.0.0.1:37875, datanodeUuid=69e1f7cc-78d0-4659-abb8-aeaa876f66db, infoPort=38747, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=345027992;c=1732071372787) 2024-11-20T02:56:15,228 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x830245ccd7fbb08b with lease ID 0x9ff821ef1d811f02: from storage DS-0472fe11-1b2f-4ca9-997f-b1908e24237b node DatanodeRegistration(127.0.0.1:37875, datanodeUuid=69e1f7cc-78d0-4659-abb8-aeaa876f66db, infoPort=38747, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=345027992;c=1732071372787), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T02:56:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x830245ccd7fbb08b with lease ID 0x9ff821ef1d811f02: Processing first storage report for DS-c5e53e88-6647-42b1-aa1e-b60db4a58bf2 from datanode DatanodeRegistration(127.0.0.1:37875, datanodeUuid=69e1f7cc-78d0-4659-abb8-aeaa876f66db, infoPort=38747, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=345027992;c=1732071372787) 2024-11-20T02:56:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x830245ccd7fbb08b with lease ID 0x9ff821ef1d811f02: from storage DS-c5e53e88-6647-42b1-aa1e-b60db4a58bf2 node DatanodeRegistration(127.0.0.1:37875, datanodeUuid=69e1f7cc-78d0-4659-abb8-aeaa876f66db, infoPort=38747, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=345027992;c=1732071372787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T02:56:15,326 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4 2024-11-20T02:56:15,445 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/zookeeper_0, clientPort=57317, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T02:56:15,454 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57317 2024-11-20T02:56:15,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:15,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:15,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741825_1001 (size=7) 2024-11-20T02:56:16,088 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b with version=8 2024-11-20T02:56:16,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/hbase-staging 2024-11-20T02:56:16,170 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T02:56:16,400 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1c82b83631a2:0 server-side Connection retries=9 2024-11-20T02:56:16,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:16,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:16,415 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:56:16,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:16,415 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:56:16,540 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T02:56:16,589 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T02:56:16,597 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T02:56:16,600 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:56:16,622 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 99856 (auto-detected) 2024-11-20T02:56:16,623 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T02:56:16,646 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36265 2024-11-20T02:56:16,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36265 connecting to ZooKeeper ensemble=127.0.0.1:57317 2024-11-20T02:56:16,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362650x0, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:56:16,811 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36265-0x101577555050000 connected 2024-11-20T02:56:16,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:16,913 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:16,924 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:56:16,929 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b, hbase.cluster.distributed=false 2024-11-20T02:56:16,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:56:16,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36265 2024-11-20T02:56:16,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36265 2024-11-20T02:56:16,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36265 2024-11-20T02:56:16,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36265 2024-11-20T02:56:16,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36265 2024-11-20T02:56:17,068 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=9 2024-11-20T02:56:17,069 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:17,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:17,070 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:56:17,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:56:17,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:56:17,073 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:56:17,075 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:56:17,076 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34099 2024-11-20T02:56:17,079 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34099 connecting to ZooKeeper ensemble=127.0.0.1:57317 2024-11-20T02:56:17,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:17,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:17,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340990x0, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:56:17,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:340990x0, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:56:17,128 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34099-0x101577555050001 connected 2024-11-20T02:56:17,132 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:56:17,142 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:56:17,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T02:56:17,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:56:17,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34099 2024-11-20T02:56:17,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34099 2024-11-20T02:56:17,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34099 2024-11-20T02:56:17,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34099 2024-11-20T02:56:17,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34099 2024-11-20T02:56:17,169 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1c82b83631a2:36265 2024-11-20T02:56:17,170 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1c82b83631a2,36265,1732071376244 2024-11-20T02:56:17,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:56:17,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:56:17,191 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1c82b83631a2,36265,1732071376244 2024-11-20T02:56:17,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T02:56:17,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:17,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:17,222 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T02:56:17,224 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1c82b83631a2,36265,1732071376244 from backup master directory 2024-11-20T02:56:17,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:56:17,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1c82b83631a2,36265,1732071376244 2024-11-20T02:56:17,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:56:17,235 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:56:17,236 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1c82b83631a2,36265,1732071376244 2024-11-20T02:56:17,238 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T02:56:17,240 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T02:56:17,294 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/hbase.id] with ID: 2ceff781-7090-4fba-8324-3a7a3899f591 2024-11-20T02:56:17,295 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/.tmp/hbase.id 2024-11-20T02:56:17,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741826_1002 (size=42) 2024-11-20T02:56:17,706 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/.tmp/hbase.id]:[hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/hbase.id] 2024-11-20T02:56:17,752 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T02:56:17,758 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T02:56:17,783 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-20T02:56:17,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:17,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:17,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741827_1003 (size=196) 2024-11-20T02:56:18,245 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:56:18,249 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T02:56:18,270 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:56:18,276 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:56:18,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741828_1004 (size=1189) 2024-11-20T02:56:18,341 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store 2024-11-20T02:56:18,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741829_1005 (size=34) 2024-11-20T02:56:18,376 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T02:56:18,380 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:18,382 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:56:18,382 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:56:18,382 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:56:18,384 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:56:18,385 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:56:18,385 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:56:18,387 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071378382Disabling compacts and flushes for region at 1732071378382Disabling writes for close at 1732071378385 (+3 ms)Writing region close event to WAL at 1732071378385Closed at 1732071378385 2024-11-20T02:56:18,390 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/.initializing 2024-11-20T02:56:18,391 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/WALs/1c82b83631a2,36265,1732071376244 2024-11-20T02:56:18,401 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:56:18,419 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C36265%2C1732071376244, suffix=, logDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/WALs/1c82b83631a2,36265,1732071376244, archiveDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/oldWALs, maxLogs=10 2024-11-20T02:56:18,443 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/WALs/1c82b83631a2,36265,1732071376244/1c82b83631a2%2C36265%2C1732071376244.1732071378424, exclude list is [], retry=0 2024-11-20T02:56:18,460 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37875,DS-0472fe11-1b2f-4ca9-997f-b1908e24237b,DISK] 2024-11-20T02:56:18,463 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T02:56:18,503 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/WALs/1c82b83631a2,36265,1732071376244/1c82b83631a2%2C36265%2C1732071376244.1732071378424 2024-11-20T02:56:18,504 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38747:38747)] 2024-11-20T02:56:18,505 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:56:18,505 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:18,509 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,510 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T02:56:18,592 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:18,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:18,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T02:56:18,601 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:18,602 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:56:18,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T02:56:18,608 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:18,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:56:18,611 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T02:56:18,618 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:18,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:56:18,621 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,626 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,627 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,639 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,640 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,652 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:56:18,658 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:56:18,666 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:56:18,668 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64151076, jitterRate=-0.04407447576522827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:56:18,682 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732071378523Initializing all the Stores at 1732071378526 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071378527 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071378528 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071378528Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071378529 (+1 ms)Cleaning up temporary data from old regions at 1732071378640 (+111 ms)Region opened successfully at 1732071378682 (+42 ms) 2024-11-20T02:56:18,695 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T02:56:18,737 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21fd2841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:56:18,776 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T02:56:18,791 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T02:56:18,791 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T02:56:18,795 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T02:56:18,798 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-20T02:56:18,804 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T02:56:18,804 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T02:56:18,858 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T02:56:18,877 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T02:56:18,938 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T02:56:18,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T02:56:18,943 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T02:56:18,955 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T02:56:18,958 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T02:56:18,963 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T02:56:18,971 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T02:56:18,973 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T02:56:18,984 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T02:56:19,005 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T02:56:19,013 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T02:56:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:56:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:56:19,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,031 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1c82b83631a2,36265,1732071376244, sessionid=0x101577555050000, setting cluster-up flag (Was=false) 2024-11-20T02:56:19,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,088 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T02:56:19,091 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,36265,1732071376244 2024-11-20T02:56:19,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:19,146 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T02:56:19,148 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,36265,1732071376244 2024-11-20T02:56:19,154 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T02:56:19,160 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(746): ClusterId : 2ceff781-7090-4fba-8324-3a7a3899f591 2024-11-20T02:56:19,163 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:56:19,238 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T02:56:19,245 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:56:19,245 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:56:19,249 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T02:56:19,257 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T02:56:19,263 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1c82b83631a2,36265,1732071376244 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T02:56:19,298 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:56:19,299 DEBUG [RS:0;1c82b83631a2:34099 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45eb3071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:56:19,299 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:56:19,300 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:56:19,300 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:56:19,300 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:56:19,300 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1c82b83631a2:0, corePoolSize=10, maxPoolSize=10 2024-11-20T02:56:19,300 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,301 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:56:19,301 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,307 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071409306 2024-11-20T02:56:19,308 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:56:19,308 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T02:56:19,308 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T02:56:19,310 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T02:56:19,313 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T02:56:19,313 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T02:56:19,314 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T02:56:19,314 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T02:56:19,315 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:19,316 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T02:56:19,316 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1c82b83631a2:34099 2024-11-20T02:56:19,316 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,323 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:56:19,323 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:56:19,323 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:56:19,325 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T02:56:19,326 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,36265,1732071376244 with port=34099, startcode=1732071377033 2024-11-20T02:56:19,327 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T02:56:19,327 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T02:56:19,330 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T02:56:19,330 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T02:56:19,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741831_1007 (size=1321) 2024-11-20T02:56:19,339 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071379332,5,FailOnTimeoutGroup] 2024-11-20T02:56:19,339 DEBUG [RS:0;1c82b83631a2:34099 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:56:19,340 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071379339,5,FailOnTimeoutGroup] 2024-11-20T02:56:19,341 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,341 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T02:56:19,342 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,343 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:56:19,392 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36265 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,395 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36265 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,414 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:56:19,415 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40175 2024-11-20T02:56:19,415 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:56:19,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:56:19,443 DEBUG [RS:0;1c82b83631a2:34099 {}] zookeeper.ZKUtil(111): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,444 WARN [RS:0;1c82b83631a2:34099 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:56:19,444 INFO [RS:0;1c82b83631a2:34099 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:56:19,444 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,447 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,34099,1732071377033] 2024-11-20T02:56:19,481 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:56:19,500 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:56:19,506 INFO [RS:0;1c82b83631a2:34099 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:56:19,507 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,508 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:56:19,515 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:56:19,517 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,517 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,518 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,518 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,518 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,518 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,518 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:56:19,519 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,519 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,519 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,519 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,519 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,520 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:56:19,520 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:56:19,520 DEBUG [RS:0;1c82b83631a2:34099 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:56:19,522 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,522 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,522 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,523 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,523 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,523 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,34099,1732071377033-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:56:19,545 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:56:19,547 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,34099,1732071377033-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,547 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,547 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.Replication(171): 1c82b83631a2,34099,1732071377033 started 2024-11-20T02:56:19,569 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:19,570 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,34099,1732071377033, RpcServer on 1c82b83631a2/172.17.0.2:34099, sessionid=0x101577555050001 2024-11-20T02:56:19,571 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:56:19,571 DEBUG [RS:0;1c82b83631a2:34099 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,571 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,34099,1732071377033' 2024-11-20T02:56:19,572 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:56:19,573 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:56:19,574 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:56:19,574 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:56:19,574 DEBUG [RS:0;1c82b83631a2:34099 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,34099,1732071377033 2024-11-20T02:56:19,574 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,34099,1732071377033' 2024-11-20T02:56:19,574 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:56:19,575 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:56:19,576 DEBUG [RS:0;1c82b83631a2:34099 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:56:19,576 INFO [RS:0;1c82b83631a2:34099 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:56:19,577 INFO [RS:0;1c82b83631a2:34099 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:56:19,733 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T02:56:19,733 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:56:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741832_1008 (size=32) 2024-11-20T02:56:20,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:20,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T02:56:20,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:56:20,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:20,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:20,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T02:56:20,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:56:20,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:20,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:20,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T02:56:20,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:56:20,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:20,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:20,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T02:56:20,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:56:20,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:20,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:20,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:56:20,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740 2024-11-20T02:56:20,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740 2024-11-20T02:56:20,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:56:20,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:56:20,172 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:56:20,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:56:20,180 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:56:20,181 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66097668, jitterRate=-0.015067994594573975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:56:20,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732071380147Initializing all the Stores at 1732071380148 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071380149 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071380149Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071380149Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071380149Cleaning up temporary data from old regions at 1732071380171 (+22 ms)Region opened successfully at 1732071380184 (+13 ms) 2024-11-20T02:56:20,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:56:20,184 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:56:20,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:56:20,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:56:20,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:56:20,186 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:56:20,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071380184Disabling compacts and flushes for region at 1732071380184Disabling writes for close at 1732071380185 (+1 ms)Writing region close event to WAL at 1732071380186 (+1 ms)Closed at 1732071380186 2024-11-20T02:56:20,189 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:56:20,189 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T02:56:20,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T02:56:20,203 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:56:20,206 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T02:56:20,358 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:21,358 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:22,359 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:23,360 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:24,361 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:25,362 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:26,362 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:26,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T02:56:26,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T02:56:27,363 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:28,364 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:29,365 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:30,366 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:31,367 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:32,368 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:33,368 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:34,369 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:35,370 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:36,371 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:37,371 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:38,372 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:39,373 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:40,374 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:41,375 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:42,375 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:43,376 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:44,377 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:45,322 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T02:56:45,377 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:46,378 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:47,379 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:48,380 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:49,381 WARN [1c82b83631a2:36265 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:56:49,591 INFO [RS:0;1c82b83631a2:34099 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:56:49,595 INFO [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C34099%2C1732071377033, suffix=, logDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033, archiveDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/oldWALs, maxLogs=32 2024-11-20T02:56:49,610 DEBUG [RS:0;1c82b83631a2:34099 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033/1c82b83631a2%2C34099%2C1732071377033.1732071409597, exclude list is [], retry=0 2024-11-20T02:56:49,614 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37875,DS-0472fe11-1b2f-4ca9-997f-b1908e24237b,DISK] 2024-11-20T02:56:49,617 INFO [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033/1c82b83631a2%2C34099%2C1732071377033.1732071409597 2024-11-20T02:56:49,617 DEBUG [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38747:38747)] 2024-11-20T02:56:49,633 DEBUG [1c82b83631a2:36265 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T02:56:49,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:49,649 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,34099,1732071377033, state=OPENING 2024-11-20T02:56:49,711 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T02:56:49,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:49,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:56:49,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:56:49,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:56:49,725 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:56:49,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:56:49,909 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T02:56:49,912 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35331, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T02:56:49,923 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T02:56:49,923 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:56:49,924 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T02:56:49,927 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C34099%2C1732071377033.meta, suffix=.meta, logDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033, archiveDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/oldWALs, maxLogs=32 2024-11-20T02:56:49,942 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033/1c82b83631a2%2C34099%2C1732071377033.meta.1732071409929.meta, exclude list is [], retry=0 2024-11-20T02:56:49,946 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37875,DS-0472fe11-1b2f-4ca9-997f-b1908e24237b,DISK] 2024-11-20T02:56:49,953 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/WALs/1c82b83631a2,34099,1732071377033/1c82b83631a2%2C34099%2C1732071377033.meta.1732071409929.meta 2024-11-20T02:56:49,954 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38747:38747)] 2024-11-20T02:56:49,954 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:56:49,956 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T02:56:49,959 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T02:56:49,963 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T02:56:49,968 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T02:56:49,969 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:49,969 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T02:56:49,969 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T02:56:49,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T02:56:49,974 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:56:49,974 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:49,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:49,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T02:56:49,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:56:49,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:49,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:49,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T02:56:49,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:56:49,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:49,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:49,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T02:56:49,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:56:49,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:49,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:56:49,984 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:56:49,986 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740 2024-11-20T02:56:49,989 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740 2024-11-20T02:56:49,992 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:56:49,992 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:56:49,993 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:56:49,996 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:56:49,998 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62992617, jitterRate=-0.061336860060691833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:56:49,999 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T02:56:50,000 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732071409969Writing region info on filesystem at 1732071409970 (+1 ms)Initializing all the Stores at 1732071409972 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071409972Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071409972Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071409972Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071409972Cleaning up temporary data from old regions at 1732071409992 (+20 ms)Running coprocessor post-open hooks at 1732071409999 (+7 ms)Region opened successfully at 1732071410000 (+1 ms) 2024-11-20T02:56:50,008 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732071409900 2024-11-20T02:56:50,021 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T02:56:50,022 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T02:56:50,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:50,027 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,34099,1732071377033, state=OPEN 2024-11-20T02:56:50,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:56:50,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:56:50,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:56:50,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:56:50,109 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:50,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T02:56:50,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,34099,1732071377033 in 381 msec 2024-11-20T02:56:50,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T02:56:50,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.9230 sec 2024-11-20T02:56:50,125 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:56:50,125 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T02:56:50,143 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:56:50,144 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,34099,1732071377033, seqNum=-1] 2024-11-20T02:56:50,162 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:56:50,165 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44587, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:56:50,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9950 sec 2024-11-20T02:56:50,185 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732071410185, completionTime=-1 2024-11-20T02:56:50,188 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T02:56:50,188 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T02:56:50,212 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T02:56:50,212 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071470212 2024-11-20T02:56:50,213 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071530213 2024-11-20T02:56:50,213 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-20T02:56:50,215 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,215 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,215 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,216 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1c82b83631a2:36265, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,216 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,217 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,222 DEBUG [master/1c82b83631a2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T02:56:50,243 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.007sec 2024-11-20T02:56:50,245 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T02:56:50,247 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T02:56:50,249 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T02:56:50,250 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T02:56:50,250 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T02:56:50,251 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:56:50,252 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T02:56:50,260 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T02:56:50,261 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T02:56:50,261 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,36265,1732071376244-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:56:50,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,312 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T02:56:50,312 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T02:56:50,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1c82b83631a2,36265,-1 for getting cluster id 2024-11-20T02:56:50,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T02:56:50,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2ceff781-7090-4fba-8324-3a7a3899f591' 2024-11-20T02:56:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T02:56:50,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2ceff781-7090-4fba-8324-3a7a3899f591" 2024-11-20T02:56:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c489ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1c82b83631a2,36265,-1] 2024-11-20T02:56:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T02:56:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:56:50,333 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T02:56:50,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1928876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:56:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,34099,1732071377033, seqNum=-1] 2024-11-20T02:56:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:56:50,346 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:56:50,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1c82b83631a2,36265,1732071376244 2024-11-20T02:56:50,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1c82b83631a2,36265,-1 for getting cluster id 2024-11-20T02:56:50,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T02:56:50,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2ceff781-7090-4fba-8324-3a7a3899f591' 2024-11-20T02:56:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T02:56:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2ceff781-7090-4fba-8324-3a7a3899f591" 2024-11-20T02:56:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab7d045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1c82b83631a2,36265,-1] 2024-11-20T02:56:50,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T02:56:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:56:50,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T02:56:50,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1080f0da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:56:50,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=405, ProcessCount=11, AvailableMemoryMB=6331 2024-11-20T02:56:50,394 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T02:56:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1c82b83631a2,36265,1732071376244 2024-11-20T02:56:50,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fad16ae 2024-11-20T02:56:50,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T02:56:50,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T02:56:50,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:56:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T02:56:50,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:56:50,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-20T02:56:50,439 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:50,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:56:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:56:50,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741835_1011 (size=428) 2024-11-20T02:56:50,863 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 55e149609e6ba0fd110650e3eeecaca2, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:56:50,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741836_1012 (size=79) 2024-11-20T02:56:51,278 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:51,279 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 55e149609e6ba0fd110650e3eeecaca2, disabling compactions & flushes 2024-11-20T02:56:51,280 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,280 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,280 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. after waiting 0 ms 2024-11-20T02:56:51,281 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,281 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,281 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 55e149609e6ba0fd110650e3eeecaca2: Waiting for close lock at 1732071411279Disabling compacts and flushes for region at 1732071411279Disabling writes for close at 1732071411280 (+1 ms)Writing region close event to WAL at 1732071411281 (+1 ms)Closed at 1732071411281 2024-11-20T02:56:51,287 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:56:51,291 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732071411287"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071411287"}]},"ts":"1732071411287"} 2024-11-20T02:56:51,296 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:56:51,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:56:51,300 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071411298"}]},"ts":"1732071411298"} 2024-11-20T02:56:51,305 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-20T02:56:51,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=55e149609e6ba0fd110650e3eeecaca2, ASSIGN}] 2024-11-20T02:56:51,309 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=55e149609e6ba0fd110650e3eeecaca2, ASSIGN 2024-11-20T02:56:51,311 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=55e149609e6ba0fd110650e3eeecaca2, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:56:51,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=55e149609e6ba0fd110650e3eeecaca2, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:56:51,471 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=55e149609e6ba0fd110650e3eeecaca2, ASSIGN because future has completed 2024-11-20T02:56:51,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55e149609e6ba0fd110650e3eeecaca2, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:56:51,638 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 55e149609e6ba0fd110650e3eeecaca2, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:56:51,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:51,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,639 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,642 INFO [StoreOpener-55e149609e6ba0fd110650e3eeecaca2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,644 INFO [StoreOpener-55e149609e6ba0fd110650e3eeecaca2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55e149609e6ba0fd110650e3eeecaca2 columnFamilyName testFamily 2024-11-20T02:56:51,644 DEBUG [StoreOpener-55e149609e6ba0fd110650e3eeecaca2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:51,645 INFO [StoreOpener-55e149609e6ba0fd110650e3eeecaca2-1 {}] regionserver.HStore(327): Store=55e149609e6ba0fd110650e3eeecaca2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:56:51,645 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,647 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,647 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,648 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,648 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,650 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,654 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:56:51,654 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 55e149609e6ba0fd110650e3eeecaca2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66303378, jitterRate=-0.012002676725387573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:56:51,655 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:56:51,656 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 55e149609e6ba0fd110650e3eeecaca2: Running coprocessor pre-open hook at 1732071411640Writing region info on filesystem at 1732071411640Initializing all the Stores at 1732071411641 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071411641Cleaning up temporary data from old regions at 1732071411648 (+7 ms)Running coprocessor post-open hooks at 1732071411655 (+7 ms)Region opened successfully at 1732071411656 (+1 ms) 2024-11-20T02:56:51,658 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2., pid=6, masterSystemTime=1732071411629 2024-11-20T02:56:51,661 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,662 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:56:51,663 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=55e149609e6ba0fd110650e3eeecaca2, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:51,667 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55e149609e6ba0fd110650e3eeecaca2, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:56:51,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T02:56:51,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 55e149609e6ba0fd110650e3eeecaca2, server=1c82b83631a2,34099,1732071377033 in 197 msec 2024-11-20T02:56:51,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T02:56:51,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=55e149609e6ba0fd110650e3eeecaca2, ASSIGN in 367 msec 2024-11-20T02:56:51,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:56:51,679 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071411678"}]},"ts":"1732071411678"} 2024-11-20T02:56:51,682 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-20T02:56:51,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:56:51,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2540 sec 2024-11-20T02:56:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:56:53,474 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-20T02:56:53,474 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-20T02:56:53,476 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:56:53,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-20T02:56:53,485 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:56:53,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-20T02:56:53,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:56:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,34099,1732071377033, seqNum=-1] 2024-11-20T02:56:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:56:53,495 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:56:53,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:56:53,517 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:56:53,521 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-20T02:56:53,525 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T02:56:53,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T02:56:53,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T02:56:53,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7946213638228934664 2024-11-20T02:56:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071416530, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:56:53,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T02:56:53,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T02:56:53,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:56:53,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:56:53,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7946213638228934666 2024-11-20T02:56:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071416563, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:56:53,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:56:53,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7946213638228934667 2024-11-20T02:56:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071416568, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:56:53,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:56:53,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7946213638228934668 2024-11-20T02:56:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071416572, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:56:53,576 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:56:53,586 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383825168_22 at /127.0.0.1:39190 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 405), ProcessCount=11 (was 11), AvailableMemoryMB=6266 (was 6331) 2024-11-20T02:56:53,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=6265 2024-11-20T02:56:53,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:56:53,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-20T02:56:53,604 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:56:53,604 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:53,604 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-20T02:56:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T02:56:53,606 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:56:53,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741837_1013 (size=414) 2024-11-20T02:56:54,019 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 283fc959520ff14da243e54e3b645eb7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:56:54,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741838_1014 (size=65) 2024-11-20T02:56:54,433 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:54,433 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 283fc959520ff14da243e54e3b645eb7, disabling compactions & flushes 2024-11-20T02:56:54,433 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,433 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,433 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. after waiting 0 ms 2024-11-20T02:56:54,433 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,433 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,434 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 283fc959520ff14da243e54e3b645eb7: Waiting for close lock at 1732071414433Disabling compacts and flushes for region at 1732071414433Disabling writes for close at 1732071414433Writing region close event to WAL at 1732071414433Closed at 1732071414433 2024-11-20T02:56:54,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:56:54,436 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732071414436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071414436"}]},"ts":"1732071414436"} 2024-11-20T02:56:54,439 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:56:54,441 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:56:54,442 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071414441"}]},"ts":"1732071414441"} 2024-11-20T02:56:54,446 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T02:56:54,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=283fc959520ff14da243e54e3b645eb7, ASSIGN}] 2024-11-20T02:56:54,448 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=283fc959520ff14da243e54e3b645eb7, ASSIGN 2024-11-20T02:56:54,450 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=283fc959520ff14da243e54e3b645eb7, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:56:54,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=283fc959520ff14da243e54e3b645eb7, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:54,611 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=283fc959520ff14da243e54e3b645eb7, ASSIGN because future has completed 2024-11-20T02:56:54,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 283fc959520ff14da243e54e3b645eb7, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:56:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T02:56:54,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 283fc959520ff14da243e54e3b645eb7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:56:54,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:56:54,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,778 INFO [StoreOpener-283fc959520ff14da243e54e3b645eb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,783 INFO [StoreOpener-283fc959520ff14da243e54e3b645eb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 283fc959520ff14da243e54e3b645eb7 columnFamilyName testFamily 2024-11-20T02:56:54,783 DEBUG [StoreOpener-283fc959520ff14da243e54e3b645eb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:56:54,784 INFO [StoreOpener-283fc959520ff14da243e54e3b645eb7-1 {}] regionserver.HStore(327): Store=283fc959520ff14da243e54e3b645eb7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:56:54,784 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,787 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,788 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:56:54,797 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 283fc959520ff14da243e54e3b645eb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70154822, jitterRate=0.04538831114768982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:56:54,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:56:54,799 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 283fc959520ff14da243e54e3b645eb7: Running coprocessor pre-open hook at 1732071414775Writing region info on filesystem at 1732071414775Initializing all the Stores at 1732071414777 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071414778 (+1 ms)Cleaning up temporary data from old regions at 1732071414789 (+11 ms)Running coprocessor post-open hooks at 1732071414798 (+9 ms)Region opened successfully at 1732071414799 (+1 ms) 2024-11-20T02:56:54,801 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., pid=9, masterSystemTime=1732071414765 2024-11-20T02:56:54,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,805 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:56:54,806 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=283fc959520ff14da243e54e3b645eb7, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:56:54,810 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 283fc959520ff14da243e54e3b645eb7, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:56:54,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T02:56:54,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 283fc959520ff14da243e54e3b645eb7, server=1c82b83631a2,34099,1732071377033 in 200 msec 2024-11-20T02:56:54,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T02:56:54,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=283fc959520ff14da243e54e3b645eb7, ASSIGN in 369 msec 2024-11-20T02:56:54,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:56:54,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071414822"}]},"ts":"1732071414822"} 2024-11-20T02:56:54,826 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T02:56:54,829 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:56:54,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2300 sec 2024-11-20T02:56:55,503 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:56:55,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-20T02:56:55,540 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T02:56:55,541 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-20T02:56:56,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T02:56:56,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T02:56:56,587 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T02:56:56,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T02:56:56,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T02:56:56,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T02:56:56,588 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-20T02:56:56,588 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T02:56:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T02:56:56,633 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-20T02:56:56,633 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T02:56:56,633 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:56:56,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T02:56:56,641 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:56:56,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-20T02:56:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:56:56,657 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:56:56,657 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T02:56:56,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T02:56:56,662 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T02:57:02,667 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-20T02:57:02,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-20T02:57:02,668 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:02,676 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=6171 (was 6265) 2024-11-20T02:57:02,685 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=6171 2024-11-20T02:57:02,688 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:02,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-20T02:57:02,692 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:02,692 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:02,692 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-20T02:57:02,694 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T02:57:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741839_1015 (size=412) 2024-11-20T02:57:03,110 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3b2cef3038851e95613f70ba8114a14, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741840_1016 (size=63) 2024-11-20T02:57:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-20T02:57:03,523 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:03,523 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f3b2cef3038851e95613f70ba8114a14, disabling compactions & flushes 2024-11-20T02:57:03,523 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. after waiting 0 ms 2024-11-20T02:57:03,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,524 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3b2cef3038851e95613f70ba8114a14: Waiting for close lock at 1732071423523Disabling compacts and flushes for region at 1732071423523Disabling writes for close at 1732071423524 (+1 ms)Writing region close event to WAL at 1732071423524Closed at 1732071423524 2024-11-20T02:57:03,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:03,528 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732071423528"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071423528"}]},"ts":"1732071423528"} 2024-11-20T02:57:03,533 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:03,535 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:03,536 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071423535"}]},"ts":"1732071423535"} 2024-11-20T02:57:03,539 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T02:57:03,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f3b2cef3038851e95613f70ba8114a14, ASSIGN}] 2024-11-20T02:57:03,542 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f3b2cef3038851e95613f70ba8114a14, ASSIGN 2024-11-20T02:57:03,544 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f3b2cef3038851e95613f70ba8114a14, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:03,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f3b2cef3038851e95613f70ba8114a14, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:03,701 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f3b2cef3038851e95613f70ba8114a14, ASSIGN because future has completed 2024-11-20T02:57:03,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3b2cef3038851e95613f70ba8114a14, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T02:57:03,870 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,870 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f3b2cef3038851e95613f70ba8114a14, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:03,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:03,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,873 INFO [StoreOpener-f3b2cef3038851e95613f70ba8114a14-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,875 INFO [StoreOpener-f3b2cef3038851e95613f70ba8114a14-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3b2cef3038851e95613f70ba8114a14 columnFamilyName testFamily 2024-11-20T02:57:03,875 DEBUG [StoreOpener-f3b2cef3038851e95613f70ba8114a14-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:03,876 INFO [StoreOpener-f3b2cef3038851e95613f70ba8114a14-1 {}] regionserver.HStore(327): Store=f3b2cef3038851e95613f70ba8114a14/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:03,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,877 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,878 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,878 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,878 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:03,885 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened f3b2cef3038851e95613f70ba8114a14; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59264914, jitterRate=-0.11688396334648132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:03,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:03,886 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f3b2cef3038851e95613f70ba8114a14: Running coprocessor pre-open hook at 1732071423871Writing region info on filesystem at 1732071423871Initializing all the Stores at 1732071423873 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071423873Cleaning up temporary data from old regions at 1732071423878 (+5 ms)Running coprocessor post-open hooks at 1732071423885 (+7 ms)Region opened successfully at 1732071423886 (+1 ms) 2024-11-20T02:57:03,887 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14., pid=12, masterSystemTime=1732071423857 2024-11-20T02:57:03,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,891 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:03,892 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f3b2cef3038851e95613f70ba8114a14, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:03,895 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3b2cef3038851e95613f70ba8114a14, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:03,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T02:57:03,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f3b2cef3038851e95613f70ba8114a14, server=1c82b83631a2,34099,1732071377033 in 195 msec 2024-11-20T02:57:03,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T02:57:03,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f3b2cef3038851e95613f70ba8114a14, ASSIGN in 360 msec 2024-11-20T02:57:03,906 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:03,906 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071423906"}]},"ts":"1732071423906"} 2024-11-20T02:57:03,909 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T02:57:03,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:03,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2230 sec 2024-11-20T02:57:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T02:57:05,722 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-20T02:57:05,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T02:57:05,722 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:05,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T02:57:05,727 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:05,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-20T02:57:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:05,743 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:05,743 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T02:57:05,747 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T02:57:05,749 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T02:57:05,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T02:57:05,752 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T02:57:06,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-20T02:57:06,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:07,543 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-20T02:57:09,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:11,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934677' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T02:57:11,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34100 deadline: 1732071437753, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934677' does not exist 2024-11-20T02:57:11,762 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1c82b83631a2,34099,1732071377033 for closing scanner id = 7946213638228934677 for f3b2cef3038851e95613f70ba8114a14 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934677' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:57:11,766 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=6130 (was 6171) 2024-11-20T02:57:11,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=6129 2024-11-20T02:57:11,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-20T02:57:11,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:11,780 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:11,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-20T02:57:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T02:57:11,781 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:11,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741841_1017 (size=423) 2024-11-20T02:57:12,198 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1583daa69102a06da07b599367f3f0d, NAME => 'testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:12,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741842_1018 (size=74) 2024-11-20T02:57:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-20T02:57:12,610 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:12,610 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing f1583daa69102a06da07b599367f3f0d, disabling compactions & flushes 2024-11-20T02:57:12,611 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. after waiting 0 ms 2024-11-20T02:57:12,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,611 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,611 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1583daa69102a06da07b599367f3f0d: Waiting for close lock at 1732071432610Disabling compacts and flushes for region at 1732071432610Disabling writes for close at 1732071432611 (+1 ms)Writing region close event to WAL at 1732071432611Closed at 1732071432611 2024-11-20T02:57:12,613 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:12,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732071432613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071432613"}]},"ts":"1732071432613"} 2024-11-20T02:57:12,618 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:12,621 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:12,621 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071432621"}]},"ts":"1732071432621"} 2024-11-20T02:57:12,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-20T02:57:12,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f1583daa69102a06da07b599367f3f0d, ASSIGN}] 2024-11-20T02:57:12,628 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f1583daa69102a06da07b599367f3f0d, ASSIGN 2024-11-20T02:57:12,629 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f1583daa69102a06da07b599367f3f0d, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:12,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f1583daa69102a06da07b599367f3f0d, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:12,790 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f1583daa69102a06da07b599367f3f0d, ASSIGN because future has completed 2024-11-20T02:57:12,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1583daa69102a06da07b599367f3f0d, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T02:57:12,957 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,958 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => f1583daa69102a06da07b599367f3f0d, NAME => 'testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:12,958 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,958 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:12,958 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,959 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,961 INFO [StoreOpener-f1583daa69102a06da07b599367f3f0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,964 INFO [StoreOpener-f1583daa69102a06da07b599367f3f0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1583daa69102a06da07b599367f3f0d columnFamilyName testFamily 2024-11-20T02:57:12,964 DEBUG [StoreOpener-f1583daa69102a06da07b599367f3f0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:12,965 INFO [StoreOpener-f1583daa69102a06da07b599367f3f0d-1 {}] regionserver.HStore(327): Store=f1583daa69102a06da07b599367f3f0d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:12,965 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,966 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,967 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,967 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,967 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,970 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,973 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:12,974 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened f1583daa69102a06da07b599367f3f0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73423691, jitterRate=0.09409825503826141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:12,974 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:12,975 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for f1583daa69102a06da07b599367f3f0d: Running coprocessor pre-open hook at 1732071432959Writing region info on filesystem at 1732071432959Initializing all the Stores at 1732071432960 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071432960Cleaning up temporary data from old regions at 1732071432967 (+7 ms)Running coprocessor post-open hooks at 1732071432974 (+7 ms)Region opened successfully at 1732071432975 (+1 ms) 2024-11-20T02:57:12,976 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d., pid=15, masterSystemTime=1732071432946 2024-11-20T02:57:12,979 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,979 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:12,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=f1583daa69102a06da07b599367f3f0d, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:12,983 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1583daa69102a06da07b599367f3f0d, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:12,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T02:57:12,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure f1583daa69102a06da07b599367f3f0d, server=1c82b83631a2,34099,1732071377033 in 193 msec 2024-11-20T02:57:12,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T02:57:12,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=f1583daa69102a06da07b599367f3f0d, ASSIGN in 361 msec 2024-11-20T02:57:12,992 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:12,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071432992"}]},"ts":"1732071432992"} 2024-11-20T02:57:12,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-20T02:57:12,995 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:12,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2190 sec 2024-11-20T02:57:13,545 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-20T02:57:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T02:57:14,814 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-20T02:57:14,815 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-20T02:57:14,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:14,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-20T02:57:14,822 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:14,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-20T02:57:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:14,836 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:14,836 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-20T02:57:14,840 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T02:57:14,843 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T02:57:14,845 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T02:57:14,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7946213638228934681 2024-11-20T02:57:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071437844, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:57:14,849 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T02:57:14,851 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T02:57:14,851 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:57:14,853 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:57:14,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=7946213638228934683 2024-11-20T02:57:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071437852, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:57:14,857 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:57:14,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=7946213638228934684 2024-11-20T02:57:14,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071437857, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:57:14,861 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:57:14,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=7946213638228934685 2024-11-20T02:57:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:34100 deadline: 1732071437861, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T02:57:14,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T02:57:14,873 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383825168_22 at /127.0.0.1:50256 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=6130 (was 6129) - AvailableMemoryMB LEAK? - 2024-11-20T02:57:14,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=6129 2024-11-20T02:57:14,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-20T02:57:14,887 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:14,887 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:14,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-20T02:57:14,889 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T02:57:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741843_1019 (size=418) 2024-11-20T02:57:15,306 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f596aa049b129099064b6456a2050df5, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:15,323 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T02:57:15,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741844_1020 (size=69) 2024-11-20T02:57:15,761 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:15,762 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing f596aa049b129099064b6456a2050df5, disabling compactions & flushes 2024-11-20T02:57:15,762 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:15,762 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:15,762 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. after waiting 0 ms 2024-11-20T02:57:15,762 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:15,762 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:15,762 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for f596aa049b129099064b6456a2050df5: Waiting for close lock at 1732071435762Disabling compacts and flushes for region at 1732071435762Disabling writes for close at 1732071435762Writing region close event to WAL at 1732071435762Closed at 1732071435762 2024-11-20T02:57:15,764 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:15,764 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732071435764"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071435764"}]},"ts":"1732071435764"} 2024-11-20T02:57:15,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:15,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:15,770 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071435769"}]},"ts":"1732071435769"} 2024-11-20T02:57:15,773 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T02:57:15,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=f596aa049b129099064b6456a2050df5, ASSIGN}] 2024-11-20T02:57:15,775 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=f596aa049b129099064b6456a2050df5, ASSIGN 2024-11-20T02:57:15,776 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=f596aa049b129099064b6456a2050df5, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T02:57:15,927 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f596aa049b129099064b6456a2050df5, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:15,930 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=f596aa049b129099064b6456a2050df5, ASSIGN because future has completed 2024-11-20T02:57:15,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f596aa049b129099064b6456a2050df5, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:16,093 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:16,093 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => f596aa049b129099064b6456a2050df5, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:16,093 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,094 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:16,094 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,094 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,096 INFO [StoreOpener-f596aa049b129099064b6456a2050df5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,099 INFO [StoreOpener-f596aa049b129099064b6456a2050df5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f596aa049b129099064b6456a2050df5 columnFamilyName testFamily 2024-11-20T02:57:16,099 DEBUG [StoreOpener-f596aa049b129099064b6456a2050df5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:16,100 INFO [StoreOpener-f596aa049b129099064b6456a2050df5-1 {}] regionserver.HStore(327): Store=f596aa049b129099064b6456a2050df5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:16,100 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,101 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,102 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,103 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,103 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,107 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,110 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:16,110 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened f596aa049b129099064b6456a2050df5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61447480, jitterRate=-0.08436119556427002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:16,111 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:16,111 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for f596aa049b129099064b6456a2050df5: Running coprocessor pre-open hook at 1732071436094Writing region info on filesystem at 1732071436094Initializing all the Stores at 1732071436096 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071436096Cleaning up temporary data from old regions at 1732071436103 (+7 ms)Running coprocessor post-open hooks at 1732071436111 (+8 ms)Region opened successfully at 1732071436111 2024-11-20T02:57:16,112 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., pid=18, masterSystemTime=1732071436083 2024-11-20T02:57:16,115 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:16,115 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:16,116 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f596aa049b129099064b6456a2050df5, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:16,119 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f596aa049b129099064b6456a2050df5, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:16,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T02:57:16,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure f596aa049b129099064b6456a2050df5, server=1c82b83631a2,34099,1732071377033 in 190 msec 2024-11-20T02:57:16,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T02:57:16,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=f596aa049b129099064b6456a2050df5, ASSIGN in 350 msec 2024-11-20T02:57:16,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:16,128 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071436127"}]},"ts":"1732071436127"} 2024-11-20T02:57:16,130 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T02:57:16,132 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:16,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2480 sec 2024-11-20T02:57:16,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-20T02:57:16,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:16,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-20T02:57:16,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T02:57:17,921 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-20T02:57:17,921 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T02:57:17,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:17,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T02:57:17,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:17,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-20T02:57:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:17,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:17,942 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T02:57:17,945 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T02:57:17,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T02:57:18,462 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:18,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-20T02:57:18,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-20T02:57:18,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:19,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-20T02:57:19,548 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-20T02:57:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:19,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T02:57:20,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=985ms, rpcTimeout=979ms 2024-11-20T02:57:20,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=985ms, rpcTimeout=979ms 2024-11-20T02:57:20,952 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:20,960 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=7175 (was 6129) - AvailableMemoryMB LEAK? - 2024-11-20T02:57:20,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=7172 2024-11-20T02:57:20,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-20T02:57:20,982 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:20,982 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:20,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-20T02:57:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T02:57:20,985 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:20,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741845_1021 (size=416) 2024-11-20T02:57:21,001 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68bbdacf9e8919980e4a0ca02727d356, NAME => 'testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:21,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741846_1022 (size=67) 2024-11-20T02:57:21,019 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:21,020 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 68bbdacf9e8919980e4a0ca02727d356, disabling compactions & flushes 2024-11-20T02:57:21,020 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,020 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,020 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. after waiting 0 ms 2024-11-20T02:57:21,020 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,020 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,020 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68bbdacf9e8919980e4a0ca02727d356: Waiting for close lock at 1732071441019Disabling compacts and flushes for region at 1732071441019Disabling writes for close at 1732071441020 (+1 ms)Writing region close event to WAL at 1732071441020Closed at 1732071441020 2024-11-20T02:57:21,023 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:21,024 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732071441023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071441023"}]},"ts":"1732071441023"} 2024-11-20T02:57:21,027 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:21,029 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:21,030 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071441029"}]},"ts":"1732071441029"} 2024-11-20T02:57:21,033 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T02:57:21,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=68bbdacf9e8919980e4a0ca02727d356, ASSIGN}] 2024-11-20T02:57:21,036 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=68bbdacf9e8919980e4a0ca02727d356, ASSIGN 2024-11-20T02:57:21,038 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=68bbdacf9e8919980e4a0ca02727d356, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:21,189 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=68bbdacf9e8919980e4a0ca02727d356, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:21,193 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=68bbdacf9e8919980e4a0ca02727d356, ASSIGN because future has completed 2024-11-20T02:57:21,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68bbdacf9e8919980e4a0ca02727d356, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:21,355 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,355 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 68bbdacf9e8919980e4a0ca02727d356, NAME => 'testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:21,356 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,356 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:21,357 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,357 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,359 INFO [StoreOpener-68bbdacf9e8919980e4a0ca02727d356-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,362 INFO [StoreOpener-68bbdacf9e8919980e4a0ca02727d356-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68bbdacf9e8919980e4a0ca02727d356 columnFamilyName testFamily 2024-11-20T02:57:21,363 DEBUG [StoreOpener-68bbdacf9e8919980e4a0ca02727d356-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:21,364 INFO [StoreOpener-68bbdacf9e8919980e4a0ca02727d356-1 {}] regionserver.HStore(327): Store=68bbdacf9e8919980e4a0ca02727d356/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:21,364 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,365 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,366 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,367 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,367 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,370 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,378 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:21,379 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 68bbdacf9e8919980e4a0ca02727d356; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65615801, jitterRate=-0.022248372435569763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:21,380 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:21,381 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 68bbdacf9e8919980e4a0ca02727d356: Running coprocessor pre-open hook at 1732071441357Writing region info on filesystem at 1732071441357Initializing all the Stores at 1732071441359 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071441359Cleaning up temporary data from old regions at 1732071441367 (+8 ms)Running coprocessor post-open hooks at 1732071441380 (+13 ms)Region opened successfully at 1732071441380 2024-11-20T02:57:21,382 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356., pid=21, masterSystemTime=1732071441348 2024-11-20T02:57:21,386 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,386 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:21,387 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=68bbdacf9e8919980e4a0ca02727d356, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:21,390 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68bbdacf9e8919980e4a0ca02727d356, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:21,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T02:57:21,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 68bbdacf9e8919980e4a0ca02727d356, server=1c82b83631a2,34099,1732071377033 in 199 msec 2024-11-20T02:57:21,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T02:57:21,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=68bbdacf9e8919980e4a0ca02727d356, ASSIGN in 362 msec 2024-11-20T02:57:21,401 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:21,402 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071441401"}]},"ts":"1732071441401"} 2024-11-20T02:57:21,405 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T02:57:21,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:21,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 429 msec 2024-11-20T02:57:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-20T02:57:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T02:57:21,991 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-20T02:57:21,991 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T02:57:21,991 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:21,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T02:57:21,996 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:21,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-20T02:57:21,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:22,012 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:22,012 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T02:57:22,015 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T02:57:22,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T02:57:22,019 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T02:57:22,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T02:57:25,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934696' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T02:57:25,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34100 deadline: 1732071446022, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934696' does not exist 2024-11-20T02:57:25,026 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1c82b83631a2,34099,1732071377033 for closing scanner id = 7946213638228934696 for 68bbdacf9e8919980e4a0ca02727d356 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934696' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:57:25,031 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1383825168_22 at /127.0.0.1:50318 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 262) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6797 (was 7172) 2024-11-20T02:57:25,038 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=6796 2024-11-20T02:57:25,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-20T02:57:25,043 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:25,043 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:25,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-20T02:57:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T02:57:25,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:25,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741847_1023 (size=411) 2024-11-20T02:57:25,455 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 48a3659d4d8ebb9407a4fe95e6cbb671, NAME => 'testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741848_1024 (size=62) 2024-11-20T02:57:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-20T02:57:25,550 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 48a3659d4d8ebb9407a4fe95e6cbb671, disabling compactions & flushes 2024-11-20T02:57:25,864 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. after waiting 0 ms 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:25,864 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:25,864 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 48a3659d4d8ebb9407a4fe95e6cbb671: Waiting for close lock at 1732071445864Disabling compacts and flushes for region at 1732071445864Disabling writes for close at 1732071445864Writing region close event to WAL at 1732071445864Closed at 1732071445864 2024-11-20T02:57:25,866 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:25,866 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732071445866"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071445866"}]},"ts":"1732071445866"} 2024-11-20T02:57:25,868 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:25,870 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:25,870 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071445870"}]},"ts":"1732071445870"} 2024-11-20T02:57:25,872 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T02:57:25,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=48a3659d4d8ebb9407a4fe95e6cbb671, ASSIGN}] 2024-11-20T02:57:25,874 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=48a3659d4d8ebb9407a4fe95e6cbb671, ASSIGN 2024-11-20T02:57:25,875 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=48a3659d4d8ebb9407a4fe95e6cbb671, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:26,027 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=48a3659d4d8ebb9407a4fe95e6cbb671, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:26,033 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=48a3659d4d8ebb9407a4fe95e6cbb671, ASSIGN because future has completed 2024-11-20T02:57:26,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48a3659d4d8ebb9407a4fe95e6cbb671, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T02:57:26,195 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:26,195 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 48a3659d4d8ebb9407a4fe95e6cbb671, NAME => 'testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:26,195 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,195 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:26,195 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,195 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,197 INFO [StoreOpener-48a3659d4d8ebb9407a4fe95e6cbb671-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,199 INFO [StoreOpener-48a3659d4d8ebb9407a4fe95e6cbb671-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48a3659d4d8ebb9407a4fe95e6cbb671 columnFamilyName testFamily 2024-11-20T02:57:26,199 DEBUG [StoreOpener-48a3659d4d8ebb9407a4fe95e6cbb671-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:26,200 INFO [StoreOpener-48a3659d4d8ebb9407a4fe95e6cbb671-1 {}] regionserver.HStore(327): Store=48a3659d4d8ebb9407a4fe95e6cbb671/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:26,200 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,201 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,202 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,203 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,203 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,205 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,208 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:26,209 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 48a3659d4d8ebb9407a4fe95e6cbb671; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71455699, jitterRate=0.06477288901805878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:26,209 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:26,210 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 48a3659d4d8ebb9407a4fe95e6cbb671: Running coprocessor pre-open hook at 1732071446196Writing region info on filesystem at 1732071446196Initializing all the Stores at 1732071446197 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071446197Cleaning up temporary data from old regions at 1732071446203 (+6 ms)Running coprocessor post-open hooks at 1732071446209 (+6 ms)Region opened successfully at 1732071446209 2024-11-20T02:57:26,211 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671., pid=24, masterSystemTime=1732071446190 2024-11-20T02:57:26,213 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:26,213 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:26,214 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=48a3659d4d8ebb9407a4fe95e6cbb671, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:26,217 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48a3659d4d8ebb9407a4fe95e6cbb671, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:26,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T02:57:26,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 48a3659d4d8ebb9407a4fe95e6cbb671, server=1c82b83631a2,34099,1732071377033 in 184 msec 2024-11-20T02:57:26,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-20T02:57:26,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=48a3659d4d8ebb9407a4fe95e6cbb671, ASSIGN in 348 msec 2024-11-20T02:57:26,225 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:26,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071446225"}]},"ts":"1732071446225"} 2024-11-20T02:57:26,228 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T02:57:26,229 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:26,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1880 sec 2024-11-20T02:57:26,858 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:26,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-20T02:57:26,892 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:26,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-20T02:57:26,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T02:57:28,072 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-20T02:57:28,072 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T02:57:28,073 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:28,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T02:57:28,082 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:28,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-20T02:57:28,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:28,097 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:28,097 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T02:57:28,100 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T02:57:28,102 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T02:57:28,103 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T02:57:28,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T02:57:30,763 INFO [master/1c82b83631a2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T02:57:30,763 INFO [master/1c82b83631a2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T02:57:31,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934700' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T02:57:31,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34100 deadline: 1732071452114, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934700' does not exist 2024-11-20T02:57:31,116 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1c82b83631a2,34099,1732071377033 for closing scanner id = 7946213638228934700 for 48a3659d4d8ebb9407a4fe95e6cbb671 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934700' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:57:31,121 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=6432 (was 6796) 2024-11-20T02:57:31,129 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=6432 2024-11-20T02:57:31,132 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:31,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T02:57:31,135 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:31,135 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:31,135 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-20T02:57:31,137 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T02:57:31,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741849_1025 (size=419) 2024-11-20T02:57:31,550 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a8a8af4417605d5b1acceda22b9dcfb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:31,552 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-20T02:57:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741850_1026 (size=70) 2024-11-20T02:57:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-20T02:57:31,962 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:31,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0a8a8af4417605d5b1acceda22b9dcfb, disabling compactions & flushes 2024-11-20T02:57:31,963 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:31,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:31,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. after waiting 0 ms 2024-11-20T02:57:31,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:31,963 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:31,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a8a8af4417605d5b1acceda22b9dcfb: Waiting for close lock at 1732071451963Disabling compacts and flushes for region at 1732071451963Disabling writes for close at 1732071451963Writing region close event to WAL at 1732071451963Closed at 1732071451963 2024-11-20T02:57:31,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:31,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732071451965"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071451965"}]},"ts":"1732071451965"} 2024-11-20T02:57:31,970 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:31,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:31,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071451972"}]},"ts":"1732071451972"} 2024-11-20T02:57:31,975 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T02:57:31,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0a8a8af4417605d5b1acceda22b9dcfb, ASSIGN}] 2024-11-20T02:57:31,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0a8a8af4417605d5b1acceda22b9dcfb, ASSIGN 2024-11-20T02:57:31,980 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0a8a8af4417605d5b1acceda22b9dcfb, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:32,131 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0a8a8af4417605d5b1acceda22b9dcfb, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:32,135 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0a8a8af4417605d5b1acceda22b9dcfb, ASSIGN because future has completed 2024-11-20T02:57:32,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a8a8af4417605d5b1acceda22b9dcfb, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T02:57:32,304 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:32,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a8a8af4417605d5b1acceda22b9dcfb, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:32,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:32,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,312 INFO [StoreOpener-0a8a8af4417605d5b1acceda22b9dcfb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,316 INFO [StoreOpener-0a8a8af4417605d5b1acceda22b9dcfb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a8a8af4417605d5b1acceda22b9dcfb columnFamilyName testFamily 2024-11-20T02:57:32,316 DEBUG [StoreOpener-0a8a8af4417605d5b1acceda22b9dcfb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:32,319 INFO [StoreOpener-0a8a8af4417605d5b1acceda22b9dcfb-1 {}] regionserver.HStore(327): Store=0a8a8af4417605d5b1acceda22b9dcfb/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:32,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,321 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,323 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,323 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,327 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,331 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:32,333 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 0a8a8af4417605d5b1acceda22b9dcfb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71291763, jitterRate=0.06233005225658417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:32,333 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:32,334 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 0a8a8af4417605d5b1acceda22b9dcfb: Running coprocessor pre-open hook at 1732071452306Writing region info on filesystem at 1732071452306Initializing all the Stores at 1732071452311 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071452312 (+1 ms)Cleaning up temporary data from old regions at 1732071452323 (+11 ms)Running coprocessor post-open hooks at 1732071452333 (+10 ms)Region opened successfully at 1732071452334 (+1 ms) 2024-11-20T02:57:32,335 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., pid=27, masterSystemTime=1732071452290 2024-11-20T02:57:32,340 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:32,340 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:32,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0a8a8af4417605d5b1acceda22b9dcfb, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:32,345 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a8a8af4417605d5b1acceda22b9dcfb, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:32,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T02:57:32,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a8a8af4417605d5b1acceda22b9dcfb, server=1c82b83631a2,34099,1732071377033 in 212 msec 2024-11-20T02:57:32,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T02:57:32,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0a8a8af4417605d5b1acceda22b9dcfb, ASSIGN in 375 msec 2024-11-20T02:57:32,357 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:32,358 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071452358"}]},"ts":"1732071452358"} 2024-11-20T02:57:32,361 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T02:57:32,362 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:32,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2300 sec 2024-11-20T02:57:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T02:57:34,173 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-20T02:57:34,173 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T02:57:34,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:34,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T02:57:34,179 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:34,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T02:57:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:34,200 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:34,200 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T02:57:34,204 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T02:57:34,206 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T02:57:36,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T02:57:36,586 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:37,832 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:37,860 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-20T02:57:40,212 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-20T02:57:40,212 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-20T02:57:40,212 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:40,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=6004 (was 6432) 2024-11-20T02:57:40,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=6004 2024-11-20T02:57:40,236 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:40,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-20T02:57:40,239 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:40,239 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:40,239 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-20T02:57:40,240 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T02:57:40,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741851_1027 (size=407) 2024-11-20T02:57:40,651 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7005b9726df1e9f513e2b6ce3acb1705, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:40,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741852_1028 (size=58) 2024-11-20T02:57:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-20T02:57:41,064 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:41,064 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 7005b9726df1e9f513e2b6ce3acb1705, disabling compactions & flushes 2024-11-20T02:57:41,065 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. after waiting 0 ms 2024-11-20T02:57:41,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,065 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,065 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7005b9726df1e9f513e2b6ce3acb1705: Waiting for close lock at 1732071461064Disabling compacts and flushes for region at 1732071461064Disabling writes for close at 1732071461065 (+1 ms)Writing region close event to WAL at 1732071461065Closed at 1732071461065 2024-11-20T02:57:41,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:41,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732071461068"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071461068"}]},"ts":"1732071461068"} 2024-11-20T02:57:41,073 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:41,074 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:41,075 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071461075"}]},"ts":"1732071461075"} 2024-11-20T02:57:41,077 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T02:57:41,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=7005b9726df1e9f513e2b6ce3acb1705, ASSIGN}] 2024-11-20T02:57:41,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=7005b9726df1e9f513e2b6ce3acb1705, ASSIGN 2024-11-20T02:57:41,080 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=7005b9726df1e9f513e2b6ce3acb1705, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:41,231 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7005b9726df1e9f513e2b6ce3acb1705, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:41,239 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=7005b9726df1e9f513e2b6ce3acb1705, ASSIGN because future has completed 2024-11-20T02:57:41,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7005b9726df1e9f513e2b6ce3acb1705, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T02:57:41,405 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 7005b9726df1e9f513e2b6ce3acb1705, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:41,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:41,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,409 INFO [StoreOpener-7005b9726df1e9f513e2b6ce3acb1705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,412 INFO [StoreOpener-7005b9726df1e9f513e2b6ce3acb1705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7005b9726df1e9f513e2b6ce3acb1705 columnFamilyName testFamily 2024-11-20T02:57:41,413 DEBUG [StoreOpener-7005b9726df1e9f513e2b6ce3acb1705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:41,414 INFO [StoreOpener-7005b9726df1e9f513e2b6ce3acb1705-1 {}] regionserver.HStore(327): Store=7005b9726df1e9f513e2b6ce3acb1705/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:41,414 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,416 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,418 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:41,422 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 7005b9726df1e9f513e2b6ce3acb1705; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63501458, jitterRate=-0.0537545382976532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:41,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:41,422 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 7005b9726df1e9f513e2b6ce3acb1705: Running coprocessor pre-open hook at 1732071461406Writing region info on filesystem at 1732071461406Initializing all the Stores at 1732071461408 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071461409 (+1 ms)Cleaning up temporary data from old regions at 1732071461416 (+7 ms)Running coprocessor post-open hooks at 1732071461422 (+6 ms)Region opened successfully at 1732071461422 2024-11-20T02:57:41,423 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705., pid=30, masterSystemTime=1732071461394 2024-11-20T02:57:41,426 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,426 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:41,427 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=7005b9726df1e9f513e2b6ce3acb1705, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:41,429 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7005b9726df1e9f513e2b6ce3acb1705, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:41,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T02:57:41,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 7005b9726df1e9f513e2b6ce3acb1705, server=1c82b83631a2,34099,1732071377033 in 191 msec 2024-11-20T02:57:41,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-20T02:57:41,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=7005b9726df1e9f513e2b6ce3acb1705, ASSIGN in 356 msec 2024-11-20T02:57:41,437 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:41,437 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071461437"}]},"ts":"1732071461437"} 2024-11-20T02:57:41,440 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T02:57:41,441 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:41,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2040 sec 2024-11-20T02:57:43,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T02:57:43,272 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-20T02:57:43,272 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T02:57:43,272 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:43,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T02:57:43,282 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:43,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-20T02:57:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:43,296 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:43,296 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T02:57:43,299 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T02:57:43,301 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T02:57:43,302 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T02:57:43,303 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T02:57:43,862 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-20T02:57:45,323 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T02:57:46,910 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:46,941 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-20T02:57:46,941 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T02:57:49,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T02:57:49,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34099 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:34100 deadline: 1732071475313, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934708' does not exist 2024-11-20T02:57:49,318 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1c82b83631a2,34099,1732071377033 for closing scanner id = 7946213638228934708 for 7005b9726df1e9f513e2b6ce3acb1705 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '7946213638228934708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:57:49,326 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=6014 (was 6004) - AvailableMemoryMB LEAK? - 2024-11-20T02:57:49,334 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=6014 2024-11-20T02:57:49,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:57:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-20T02:57:49,339 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:57:49,339 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:49,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-20T02:57:49,341 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:57:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T02:57:49,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741853_1029 (size=423) 2024-11-20T02:57:49,755 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9dc19e9ae8ec333d096a4151ce9cd7b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b 2024-11-20T02:57:49,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741854_1030 (size=74) 2024-11-20T02:57:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-20T02:57:50,166 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:50,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9dc19e9ae8ec333d096a4151ce9cd7b2, disabling compactions & flushes 2024-11-20T02:57:50,167 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. after waiting 0 ms 2024-11-20T02:57:50,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,167 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,167 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9dc19e9ae8ec333d096a4151ce9cd7b2: Waiting for close lock at 1732071470167Disabling compacts and flushes for region at 1732071470167Disabling writes for close at 1732071470167Writing region close event to WAL at 1732071470167Closed at 1732071470167 2024-11-20T02:57:50,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:57:50,169 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732071470169"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071470169"}]},"ts":"1732071470169"} 2024-11-20T02:57:50,172 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:57:50,173 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:57:50,173 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071470173"}]},"ts":"1732071470173"} 2024-11-20T02:57:50,175 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T02:57:50,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9dc19e9ae8ec333d096a4151ce9cd7b2, ASSIGN}] 2024-11-20T02:57:50,177 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9dc19e9ae8ec333d096a4151ce9cd7b2, ASSIGN 2024-11-20T02:57:50,178 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9dc19e9ae8ec333d096a4151ce9cd7b2, ASSIGN; state=OFFLINE, location=1c82b83631a2,34099,1732071377033; forceNewPlan=false, retain=false 2024-11-20T02:57:50,216 DEBUG [master/1c82b83631a2:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T02:57:50,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9dc19e9ae8ec333d096a4151ce9cd7b2, regionState=OPENING, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:50,334 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9dc19e9ae8ec333d096a4151ce9cd7b2, ASSIGN because future has completed 2024-11-20T02:57:50,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dc19e9ae8ec333d096a4151ce9cd7b2, server=1c82b83631a2,34099,1732071377033}] 2024-11-20T02:57:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T02:57:50,495 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,495 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dc19e9ae8ec333d096a4151ce9cd7b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:57:50,495 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,495 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:57:50,496 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,496 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,498 INFO [StoreOpener-9dc19e9ae8ec333d096a4151ce9cd7b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,500 INFO [StoreOpener-9dc19e9ae8ec333d096a4151ce9cd7b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dc19e9ae8ec333d096a4151ce9cd7b2 columnFamilyName testFamily 2024-11-20T02:57:50,500 DEBUG [StoreOpener-9dc19e9ae8ec333d096a4151ce9cd7b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:57:50,500 INFO [StoreOpener-9dc19e9ae8ec333d096a4151ce9cd7b2-1 {}] regionserver.HStore(327): Store=9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:57:50,501 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,502 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,502 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,503 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,503 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,505 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,508 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:57:50,509 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9dc19e9ae8ec333d096a4151ce9cd7b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70080150, jitterRate=0.04427561163902283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:57:50,509 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:50,510 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9dc19e9ae8ec333d096a4151ce9cd7b2: Running coprocessor pre-open hook at 1732071470496Writing region info on filesystem at 1732071470496Initializing all the Stores at 1732071470497 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071470497Cleaning up temporary data from old regions at 1732071470503 (+6 ms)Running coprocessor post-open hooks at 1732071470509 (+6 ms)Region opened successfully at 1732071470510 (+1 ms) 2024-11-20T02:57:50,511 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., pid=33, masterSystemTime=1732071470490 2024-11-20T02:57:50,514 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,514 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:50,515 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9dc19e9ae8ec333d096a4151ce9cd7b2, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34099,1732071377033 2024-11-20T02:57:50,517 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dc19e9ae8ec333d096a4151ce9cd7b2, server=1c82b83631a2,34099,1732071377033 because future has completed 2024-11-20T02:57:50,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-20T02:57:50,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dc19e9ae8ec333d096a4151ce9cd7b2, server=1c82b83631a2,34099,1732071377033 in 184 msec 2024-11-20T02:57:50,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T02:57:50,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9dc19e9ae8ec333d096a4151ce9cd7b2, ASSIGN in 346 msec 2024-11-20T02:57:50,524 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:57:50,524 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071470524"}]},"ts":"1732071470524"} 2024-11-20T02:57:50,526 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T02:57:50,527 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:57:50,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1900 sec 2024-11-20T02:57:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36265 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T02:57:52,362 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-20T02:57:52,362 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T02:57:52,363 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:52,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T02:57:52,371 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:57:52,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T02:57:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:52,386 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T02:57:52,386 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T02:57:52,389 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T02:57:52,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T02:57:53,392 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T02:57:53,393 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T02:57:53,393 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-20T02:57:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2] 2024-11-20T02:57:54,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34099 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T02:57:55,392 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1c82b83631a2:34099 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=979ms 2024-11-20T02:57:55,392 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=979ms 2024-11-20T02:57:55,392 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., hostname=1c82b83631a2,34099,1732071377033, seqNum=2 from cache 2024-11-20T02:57:55,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=211 (was 221), ProcessCount=11 (was 11), AvailableMemoryMB=6005 (was 6014) 2024-11-20T02:57:55,401 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:57:55,401 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:57:55,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T02:57:55,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T02:57:55,407 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:57:55,407 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:57:55,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T02:57:55,407 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T02:57:55,407 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1030710501, stopped=false 2024-11-20T02:57:55,407 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1c82b83631a2,36265,1732071376244 2024-11-20T02:57:55,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:57:55,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:57:55,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:57:55,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:57:55,471 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:57:55,472 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:57:55,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:57:55,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:57:55,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:57:55,474 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,34099,1732071377033' ***** 2024-11-20T02:57:55,474 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T02:57:55,475 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:57:55,475 INFO [RS:0;1c82b83631a2:34099 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:57:55,475 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:57:55,476 INFO [RS:0;1c82b83631a2:34099 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:57:55,476 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 48a3659d4d8ebb9407a4fe95e6cbb671 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 68bbdacf9e8919980e4a0ca02727d356 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 9dc19e9ae8ec333d096a4151ce9cd7b2 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 7005b9726df1e9f513e2b6ce3acb1705 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for f3b2cef3038851e95613f70ba8114a14 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for f1583daa69102a06da07b599367f3f0d 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for f596aa049b129099064b6456a2050df5 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a8a8af4417605d5b1acceda22b9dcfb 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 283fc959520ff14da243e54e3b645eb7 2024-11-20T02:57:55,477 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48a3659d4d8ebb9407a4fe95e6cbb671, disabling compactions & flushes 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:55,477 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,34099,1732071377033 2024-11-20T02:57:55,477 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:55,477 INFO [RS:0;1c82b83631a2:34099 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:57:55,477 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. after waiting 0 ms 2024-11-20T02:57:55,477 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:55,478 INFO [RS:0;1c82b83631a2:34099 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1c82b83631a2:34099. 2024-11-20T02:57:55,478 DEBUG [RS:0;1c82b83631a2:34099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:57:55,478 DEBUG [RS:0;1c82b83631a2:34099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:55,478 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:57:55,478 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:57:55,478 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:57:55,478 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T02:57:55,479 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-20T02:57:55,479 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:57:55,479 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:57:55,479 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:57:55,479 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 48a3659d4d8ebb9407a4fe95e6cbb671, 55e149609e6ba0fd110650e3eeecaca2, 68bbdacf9e8919980e4a0ca02727d356, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:55,479 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:57:55,479 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:57:55,481 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 48a3659d4d8ebb9407a4fe95e6cbb671 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:55,481 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-20T02:57:55,522 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T02:57:55,523 INFO [regionserver/1c82b83631a2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T02:57:55,534 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/.tmp/testFamily/f18928a6e5434e60a3c69b27f6e0e225 is 61, key is row-0/testFamily:testQualifier/1732071448088/Put/seqid=0 2024-11-20T02:57:55,541 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/info/567d51dda66048d79a17550133ac3d18 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2./info:regioninfo/1732071411662/Put/seqid=0 2024-11-20T02:57:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741855_1031 (size=5049) 2024-11-20T02:57:55,547 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/.tmp/testFamily/f18928a6e5434e60a3c69b27f6e0e225 2024-11-20T02:57:55,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741856_1032 (size=20564) 2024-11-20T02:57:55,592 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/.tmp/testFamily/f18928a6e5434e60a3c69b27f6e0e225 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/testFamily/f18928a6e5434e60a3c69b27f6e0e225 2024-11-20T02:57:55,602 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/testFamily/f18928a6e5434e60a3c69b27f6e0e225, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:55,611 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 48a3659d4d8ebb9407a4fe95e6cbb671 in 129ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:55,618 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNext/48a3659d4d8ebb9407a4fe95e6cbb671/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:55,620 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:55,620 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48a3659d4d8ebb9407a4fe95e6cbb671: Waiting for close lock at 1732071475477Running coprocessor pre-close hooks at 1732071475477Disabling compacts and flushes for region at 1732071475477Disabling writes for close at 1732071475477Obtaining lock to block concurrent updates at 1732071475481 (+4 ms)Preparing flush snapshotting stores in 48a3659d4d8ebb9407a4fe95e6cbb671 at 1732071475481Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071475488 (+7 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. at 1732071475489 (+1 ms)Flushing 48a3659d4d8ebb9407a4fe95e6cbb671/testFamily: creating writer at 1732071475491 (+2 ms)Flushing 48a3659d4d8ebb9407a4fe95e6cbb671/testFamily: appending metadata at 1732071475529 (+38 ms)Flushing 48a3659d4d8ebb9407a4fe95e6cbb671/testFamily: closing flushed file at 1732071475531 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b0ffc03: reopening flushed file at 1732071475590 (+59 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 48a3659d4d8ebb9407a4fe95e6cbb671 in 129ms, sequenceid=8, compaction requested=false at 1732071475611 (+21 ms)Writing region close event to WAL at 1732071475612 (+1 ms)Running coprocessor post-close hooks at 1732071475619 (+7 ms)Closed at 1732071475620 (+1 ms) 2024-11-20T02:57:55,621 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732071445040.48a3659d4d8ebb9407a4fe95e6cbb671. 2024-11-20T02:57:55,621 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 68bbdacf9e8919980e4a0ca02727d356, disabling compactions & flushes 2024-11-20T02:57:55,621 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:55,621 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:55,621 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. after waiting 0 ms 2024-11-20T02:57:55,621 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:55,621 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 68bbdacf9e8919980e4a0ca02727d356 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:55,641 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/.tmp/testFamily/355a9ffbb7af4716b7c9348e82fb2da2 is 61, key is row-0/testFamily:testQualifier/1732071442001/Put/seqid=0 2024-11-20T02:57:55,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741857_1033 (size=5049) 2024-11-20T02:57:55,679 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 68bbdacf9e8919980e4a0ca02727d356, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:55,880 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 68bbdacf9e8919980e4a0ca02727d356, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:55,913 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-20T02:57:55,949 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/info/567d51dda66048d79a17550133ac3d18 2024-11-20T02:57:55,973 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/ns/a1031b5b2e954139a4b367c8a810cddb is 43, key is default/ns:d/1732071410169/Put/seqid=0 2024-11-20T02:57:55,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741858_1034 (size=5153) 2024-11-20T02:57:55,997 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:57:56,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-20T02:57:56,047 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/.tmp/testFamily/355a9ffbb7af4716b7c9348e82fb2da2 2024-11-20T02:57:56,056 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/.tmp/testFamily/355a9ffbb7af4716b7c9348e82fb2da2 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/testFamily/355a9ffbb7af4716b7c9348e82fb2da2 2024-11-20T02:57:56,065 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/testFamily/355a9ffbb7af4716b7c9348e82fb2da2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:56,066 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 68bbdacf9e8919980e4a0ca02727d356 in 445ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:56,072 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnNextAsync/68bbdacf9e8919980e4a0ca02727d356/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:56,073 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 68bbdacf9e8919980e4a0ca02727d356: Waiting for close lock at 1732071475621Running coprocessor pre-close hooks at 1732071475621Disabling compacts and flushes for region at 1732071475621Disabling writes for close at 1732071475621Obtaining lock to block concurrent updates at 1732071475621Preparing flush snapshotting stores in 68bbdacf9e8919980e4a0ca02727d356 at 1732071475621Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071475621Flushing stores of testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. at 1732071475622 (+1 ms)Flushing 68bbdacf9e8919980e4a0ca02727d356/testFamily: creating writer at 1732071475622Flushing 68bbdacf9e8919980e4a0ca02727d356/testFamily: appending metadata at 1732071475640 (+18 ms)Flushing 68bbdacf9e8919980e4a0ca02727d356/testFamily: closing flushed file at 1732071475640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53ad7d60: reopening flushed file at 1732071476055 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 68bbdacf9e8919980e4a0ca02727d356 in 445ms, sequenceid=8, compaction requested=false at 1732071476066 (+11 ms)Writing region close event to WAL at 1732071476067 (+1 ms)Running coprocessor post-close hooks at 1732071476073 (+6 ms)Closed at 1732071476073 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732071440977.68bbdacf9e8919980e4a0ca02727d356. 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9dc19e9ae8ec333d096a4151ce9cd7b2, disabling compactions & flushes 2024-11-20T02:57:56,073 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. after waiting 0 ms 2024-11-20T02:57:56,073 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:56,074 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9dc19e9ae8ec333d096a4151ce9cd7b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:56,080 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:56,090 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/.tmp/testFamily/20fbe53e258f402192fef37365db2260 is 61, key is row-0/testFamily:testQualifier/1732071472376/Put/seqid=0 2024-11-20T02:57:56,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741859_1035 (size=5049) 2024-11-20T02:57:56,280 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:56,381 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/ns/a1031b5b2e954139a4b367c8a810cddb 2024-11-20T02:57:56,411 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/table/9baf1b6b098d4a78952005d1bf4f8040 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732071411678/Put/seqid=0 2024-11-20T02:57:56,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741860_1036 (size=6699) 2024-11-20T02:57:56,480 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-20T02:57:56,481 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1325): Online Regions={9dc19e9ae8ec333d096a4151ce9cd7b2=testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., 7005b9726df1e9f513e2b6ce3acb1705=hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705., f3b2cef3038851e95613f70ba8114a14=hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14., 1588230740=hbase:meta,,1.1588230740, f1583daa69102a06da07b599367f3f0d=testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d., f596aa049b129099064b6456a2050df5=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., 0a8a8af4417605d5b1acceda22b9dcfb=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., 283fc959520ff14da243e54e3b645eb7=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., 55e149609e6ba0fd110650e3eeecaca2=testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.} 2024-11-20T02:57:56,481 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 7005b9726df1e9f513e2b6ce3acb1705, 9dc19e9ae8ec333d096a4151ce9cd7b2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:56,496 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/.tmp/testFamily/20fbe53e258f402192fef37365db2260 2024-11-20T02:57:56,509 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/.tmp/testFamily/20fbe53e258f402192fef37365db2260 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily/20fbe53e258f402192fef37365db2260 2024-11-20T02:57:56,518 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily/20fbe53e258f402192fef37365db2260, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:56,519 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9dc19e9ae8ec333d096a4151ce9cd7b2 in 446ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:56,524 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScannerAsync/9dc19e9ae8ec333d096a4151ce9cd7b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:56,525 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9dc19e9ae8ec333d096a4151ce9cd7b2: Waiting for close lock at 1732071476073Running coprocessor pre-close hooks at 1732071476073Disabling compacts and flushes for region at 1732071476073Disabling writes for close at 1732071476073Obtaining lock to block concurrent updates at 1732071476074 (+1 ms)Preparing flush snapshotting stores in 9dc19e9ae8ec333d096a4151ce9cd7b2 at 1732071476074Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071476074Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. at 1732071476075 (+1 ms)Flushing 9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily: creating writer at 1732071476075Flushing 9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily: appending metadata at 1732071476089 (+14 ms)Flushing 9dc19e9ae8ec333d096a4151ce9cd7b2/testFamily: closing flushed file at 1732071476090 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d1206c3: reopening flushed file at 1732071476507 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9dc19e9ae8ec333d096a4151ce9cd7b2 in 446ms, sequenceid=8, compaction requested=false at 1732071476519 (+12 ms)Writing region close event to WAL at 1732071476520 (+1 ms)Running coprocessor post-close hooks at 1732071476524 (+4 ms)Closed at 1732071476525 (+1 ms) 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732071469336.9dc19e9ae8ec333d096a4151ce9cd7b2. 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7005b9726df1e9f513e2b6ce3acb1705, disabling compactions & flushes 2024-11-20T02:57:56,525 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. after waiting 0 ms 2024-11-20T02:57:56,525 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:56,525 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7005b9726df1e9f513e2b6ce3acb1705 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:56,540 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/.tmp/testFamily/395f18e479e141ee85a7a27842d08e8a is 61, key is row-0/testFamily:testQualifier/1732071463287/Put/seqid=0 2024-11-20T02:57:56,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741861_1037 (size=5049) 2024-11-20T02:57:56,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-20T02:57:56,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-20T02:57:56,681 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 1588230740, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 7005b9726df1e9f513e2b6ce3acb1705, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:56,819 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/table/9baf1b6b098d4a78952005d1bf4f8040 2024-11-20T02:57:56,831 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/info/567d51dda66048d79a17550133ac3d18 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/info/567d51dda66048d79a17550133ac3d18 2024-11-20T02:57:56,840 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/info/567d51dda66048d79a17550133ac3d18, entries=100, sequenceid=56, filesize=20.1 K 2024-11-20T02:57:56,841 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/ns/a1031b5b2e954139a4b367c8a810cddb as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/ns/a1031b5b2e954139a4b367c8a810cddb 2024-11-20T02:57:56,848 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/ns/a1031b5b2e954139a4b367c8a810cddb, entries=2, sequenceid=56, filesize=5.0 K 2024-11-20T02:57:56,850 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/.tmp/table/9baf1b6b098d4a78952005d1bf4f8040 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/table/9baf1b6b098d4a78952005d1bf4f8040 2024-11-20T02:57:56,858 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/table/9baf1b6b098d4a78952005d1bf4f8040, entries=20, sequenceid=56, filesize=6.5 K 2024-11-20T02:57:56,859 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1380ms, sequenceid=56, compaction requested=false 2024-11-20T02:57:56,865 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-20T02:57:56,866 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T02:57:56,866 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:57:56,866 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071475479Running coprocessor pre-close hooks at 1732071475479Disabling compacts and flushes for region at 1732071475479Disabling writes for close at 1732071475479Obtaining lock to block concurrent updates at 1732071475481 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732071475481Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732071475488 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732071475489 (+1 ms)Flushing 1588230740/info: creating writer at 1732071475491 (+2 ms)Flushing 1588230740/info: appending metadata at 1732071475536 (+45 ms)Flushing 1588230740/info: closing flushed file at 1732071475536Flushing 1588230740/ns: creating writer at 1732071475958 (+422 ms)Flushing 1588230740/ns: appending metadata at 1732071475972 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732071475972Flushing 1588230740/table: creating writer at 1732071476394 (+422 ms)Flushing 1588230740/table: appending metadata at 1732071476410 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732071476410Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69662924: reopening flushed file at 1732071476830 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ffce242: reopening flushed file at 1732071476840 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eaa037f: reopening flushed file at 1732071476849 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1380ms, sequenceid=56, compaction requested=false at 1732071476860 (+11 ms)Writing region close event to WAL at 1732071476861 (+1 ms)Running coprocessor post-close hooks at 1732071476866 (+5 ms)Closed at 1732071476866 2024-11-20T02:57:56,867 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T02:57:56,881 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, 7005b9726df1e9f513e2b6ce3acb1705, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:56,947 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/.tmp/testFamily/395f18e479e141ee85a7a27842d08e8a 2024-11-20T02:57:56,962 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/.tmp/testFamily/395f18e479e141ee85a7a27842d08e8a as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/testFamily/395f18e479e141ee85a7a27842d08e8a 2024-11-20T02:57:56,970 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/testFamily/395f18e479e141ee85a7a27842d08e8a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:56,971 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7005b9726df1e9f513e2b6ce3acb1705 in 446ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:56,977 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNext/7005b9726df1e9f513e2b6ce3acb1705/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:56,978 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:56,978 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7005b9726df1e9f513e2b6ce3acb1705: Waiting for close lock at 1732071476525Running coprocessor pre-close hooks at 1732071476525Disabling compacts and flushes for region at 1732071476525Disabling writes for close at 1732071476525Obtaining lock to block concurrent updates at 1732071476525Preparing flush snapshotting stores in 7005b9726df1e9f513e2b6ce3acb1705 at 1732071476525Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071476525Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. at 1732071476526 (+1 ms)Flushing 7005b9726df1e9f513e2b6ce3acb1705/testFamily: creating writer at 1732071476526Flushing 7005b9726df1e9f513e2b6ce3acb1705/testFamily: appending metadata at 1732071476539 (+13 ms)Flushing 7005b9726df1e9f513e2b6ce3acb1705/testFamily: closing flushed file at 1732071476539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60448b23: reopening flushed file at 1732071476961 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7005b9726df1e9f513e2b6ce3acb1705 in 446ms, sequenceid=8, compaction requested=false at 1732071476971 (+10 ms)Writing region close event to WAL at 1732071476972 (+1 ms)Running coprocessor post-close hooks at 1732071476978 (+6 ms)Closed at 1732071476978 2024-11-20T02:57:56,978 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732071460236.7005b9726df1e9f513e2b6ce3acb1705. 2024-11-20T02:57:56,978 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f3b2cef3038851e95613f70ba8114a14, disabling compactions & flushes 2024-11-20T02:57:56,979 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:56,979 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:56,979 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. after waiting 0 ms 2024-11-20T02:57:56,979 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:56,979 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f3b2cef3038851e95613f70ba8114a14 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:56,996 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/.tmp/testFamily/e605a75bbab944f6ad241b8f08adbeeb is 61, key is row-0/testFamily:testQualifier/1732071425732/Put/seqid=0 2024-11-20T02:57:57,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741862_1038 (size=5049) 2024-11-20T02:57:57,082 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:57,282 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f1583daa69102a06da07b599367f3f0d, f3b2cef3038851e95613f70ba8114a14, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:57,402 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/.tmp/testFamily/e605a75bbab944f6ad241b8f08adbeeb 2024-11-20T02:57:57,445 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/.tmp/testFamily/e605a75bbab944f6ad241b8f08adbeeb as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/testFamily/e605a75bbab944f6ad241b8f08adbeeb 2024-11-20T02:57:57,455 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/testFamily/e605a75bbab944f6ad241b8f08adbeeb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:57,457 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f3b2cef3038851e95613f70ba8114a14 in 478ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:57,463 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnNextAsync/f3b2cef3038851e95613f70ba8114a14/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:57,464 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:57,464 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f3b2cef3038851e95613f70ba8114a14: Waiting for close lock at 1732071476978Running coprocessor pre-close hooks at 1732071476978Disabling compacts and flushes for region at 1732071476978Disabling writes for close at 1732071476979 (+1 ms)Obtaining lock to block concurrent updates at 1732071476979Preparing flush snapshotting stores in f3b2cef3038851e95613f70ba8114a14 at 1732071476979Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071476979Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. at 1732071476980 (+1 ms)Flushing f3b2cef3038851e95613f70ba8114a14/testFamily: creating writer at 1732071476980Flushing f3b2cef3038851e95613f70ba8114a14/testFamily: appending metadata at 1732071476996 (+16 ms)Flushing f3b2cef3038851e95613f70ba8114a14/testFamily: closing flushed file at 1732071476996Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a095aee: reopening flushed file at 1732071477444 (+448 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f3b2cef3038851e95613f70ba8114a14 in 478ms, sequenceid=8, compaction requested=false at 1732071477457 (+13 ms)Writing region close event to WAL at 1732071477458 (+1 ms)Running coprocessor post-close hooks at 1732071477464 (+6 ms)Closed at 1732071477464 2024-11-20T02:57:57,464 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732071422687.f3b2cef3038851e95613f70ba8114a14. 2024-11-20T02:57:57,465 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1583daa69102a06da07b599367f3f0d, disabling compactions & flushes 2024-11-20T02:57:57,465 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:57,465 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:57,465 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. after waiting 0 ms 2024-11-20T02:57:57,465 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:57,465 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f1583daa69102a06da07b599367f3f0d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:57,482 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/.tmp/testFamily/76475649260949bdb42ad1fecaea2d2d is 61, key is row-0/testFamily:testQualifier/1732071434827/Put/seqid=0 2024-11-20T02:57:57,482 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-20T02:57:57,482 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1325): Online Regions={f1583daa69102a06da07b599367f3f0d=testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d., f596aa049b129099064b6456a2050df5=testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., 0a8a8af4417605d5b1acceda22b9dcfb=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., 283fc959520ff14da243e54e3b645eb7=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., 55e149609e6ba0fd110650e3eeecaca2=testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.} 2024-11-20T02:57:57,483 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f1583daa69102a06da07b599367f3f0d, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741863_1039 (size=5049) 2024-11-20T02:57:57,683 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f1583daa69102a06da07b599367f3f0d, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:57,883 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f1583daa69102a06da07b599367f3f0d, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:57,890 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/.tmp/testFamily/76475649260949bdb42ad1fecaea2d2d 2024-11-20T02:57:57,906 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/.tmp/testFamily/76475649260949bdb42ad1fecaea2d2d as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/testFamily/76475649260949bdb42ad1fecaea2d2d 2024-11-20T02:57:57,914 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/testFamily/76475649260949bdb42ad1fecaea2d2d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:57,916 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f1583daa69102a06da07b599367f3f0d in 451ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:57,920 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextException/f1583daa69102a06da07b599367f3f0d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:57,921 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:57,921 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1583daa69102a06da07b599367f3f0d: Waiting for close lock at 1732071477464Running coprocessor pre-close hooks at 1732071477464Disabling compacts and flushes for region at 1732071477464Disabling writes for close at 1732071477465 (+1 ms)Obtaining lock to block concurrent updates at 1732071477465Preparing flush snapshotting stores in f1583daa69102a06da07b599367f3f0d at 1732071477465Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071477465Flushing stores of testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. at 1732071477466 (+1 ms)Flushing f1583daa69102a06da07b599367f3f0d/testFamily: creating writer at 1732071477466Flushing f1583daa69102a06da07b599367f3f0d/testFamily: appending metadata at 1732071477482 (+16 ms)Flushing f1583daa69102a06da07b599367f3f0d/testFamily: closing flushed file at 1732071477482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52a987ea: reopening flushed file at 1732071477904 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f1583daa69102a06da07b599367f3f0d in 451ms, sequenceid=8, compaction requested=false at 1732071477916 (+12 ms)Writing region close event to WAL at 1732071477916Running coprocessor post-close hooks at 1732071477921 (+5 ms)Closed at 1732071477921 2024-11-20T02:57:57,921 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732071431776.f1583daa69102a06da07b599367f3f0d. 2024-11-20T02:57:57,921 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f596aa049b129099064b6456a2050df5, disabling compactions & flushes 2024-11-20T02:57:57,922 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. after waiting 0 ms 2024-11-20T02:57:57,922 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:57,922 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f596aa049b129099064b6456a2050df5 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:57,938 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/.tmp/testFamily/b7adca27e4084b55a5acf126a5618689 is 61, key is row-0/testFamily:testQualifier/1732071437931/Put/seqid=0 2024-11-20T02:57:57,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741864_1040 (size=5049) 2024-11-20T02:57:58,084 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:58,284 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2, f596aa049b129099064b6456a2050df5 2024-11-20T02:57:58,345 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/.tmp/testFamily/b7adca27e4084b55a5acf126a5618689 2024-11-20T02:57:58,361 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/.tmp/testFamily/b7adca27e4084b55a5acf126a5618689 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/testFamily/b7adca27e4084b55a5acf126a5618689 2024-11-20T02:57:58,370 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/testFamily/b7adca27e4084b55a5acf126a5618689, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:58,371 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f596aa049b129099064b6456a2050df5 in 449ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:58,377 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testNormalScanTimeoutOnOpenScanner/f596aa049b129099064b6456a2050df5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:58,378 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:58,378 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f596aa049b129099064b6456a2050df5: Waiting for close lock at 1732071477921Running coprocessor pre-close hooks at 1732071477921Disabling compacts and flushes for region at 1732071477921Disabling writes for close at 1732071477922 (+1 ms)Obtaining lock to block concurrent updates at 1732071477922Preparing flush snapshotting stores in f596aa049b129099064b6456a2050df5 at 1732071477922Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071477922Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. at 1732071477923 (+1 ms)Flushing f596aa049b129099064b6456a2050df5/testFamily: creating writer at 1732071477923Flushing f596aa049b129099064b6456a2050df5/testFamily: appending metadata at 1732071477937 (+14 ms)Flushing f596aa049b129099064b6456a2050df5/testFamily: closing flushed file at 1732071477937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@179a260b: reopening flushed file at 1732071478360 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f596aa049b129099064b6456a2050df5 in 449ms, sequenceid=8, compaction requested=false at 1732071478371 (+11 ms)Writing region close event to WAL at 1732071478372 (+1 ms)Running coprocessor post-close hooks at 1732071478378 (+6 ms)Closed at 1732071478378 2024-11-20T02:57:58,378 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732071434883.f596aa049b129099064b6456a2050df5. 2024-11-20T02:57:58,379 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a8a8af4417605d5b1acceda22b9dcfb, disabling compactions & flushes 2024-11-20T02:57:58,379 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:58,379 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:58,379 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. after waiting 0 ms 2024-11-20T02:57:58,379 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:58,379 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0a8a8af4417605d5b1acceda22b9dcfb 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:58,401 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/.tmp/testFamily/22e70092a46b47af94da58c4adebcb3e is 61, key is row-0/testFamily:testQualifier/1732071454184/Put/seqid=0 2024-11-20T02:57:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741865_1041 (size=5049) 2024-11-20T02:57:58,484 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-20T02:57:58,484 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1325): Online Regions={0a8a8af4417605d5b1acceda22b9dcfb=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., 283fc959520ff14da243e54e3b645eb7=hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., 55e149609e6ba0fd110650e3eeecaca2=testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.} 2024-11-20T02:57:58,485 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:58,685 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 0a8a8af4417605d5b1acceda22b9dcfb, 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:58,807 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/.tmp/testFamily/22e70092a46b47af94da58c4adebcb3e 2024-11-20T02:57:58,817 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/.tmp/testFamily/22e70092a46b47af94da58c4adebcb3e as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/testFamily/22e70092a46b47af94da58c4adebcb3e 2024-11-20T02:57:58,826 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/testFamily/22e70092a46b47af94da58c4adebcb3e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:58,827 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a8a8af4417605d5b1acceda22b9dcfb in 448ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:58,832 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0a8a8af4417605d5b1acceda22b9dcfb/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:58,833 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a8a8af4417605d5b1acceda22b9dcfb: Waiting for close lock at 1732071478379Running coprocessor pre-close hooks at 1732071478379Disabling compacts and flushes for region at 1732071478379Disabling writes for close at 1732071478379Obtaining lock to block concurrent updates at 1732071478379Preparing flush snapshotting stores in 0a8a8af4417605d5b1acceda22b9dcfb at 1732071478379Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071478379Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. at 1732071478380 (+1 ms)Flushing 0a8a8af4417605d5b1acceda22b9dcfb/testFamily: creating writer at 1732071478380Flushing 0a8a8af4417605d5b1acceda22b9dcfb/testFamily: appending metadata at 1732071478401 (+21 ms)Flushing 0a8a8af4417605d5b1acceda22b9dcfb/testFamily: closing flushed file at 1732071478401Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ede8628: reopening flushed file at 1732071478816 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a8a8af4417605d5b1acceda22b9dcfb in 448ms, sequenceid=8, compaction requested=false at 1732071478827 (+11 ms)Writing region close event to WAL at 1732071478828 (+1 ms)Running coprocessor post-close hooks at 1732071478833 (+5 ms)Closed at 1732071478833 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732071451131.0a8a8af4417605d5b1acceda22b9dcfb. 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 283fc959520ff14da243e54e3b645eb7, disabling compactions & flushes 2024-11-20T02:57:58,833 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. after waiting 0 ms 2024-11-20T02:57:58,833 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:57:58,833 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 283fc959520ff14da243e54e3b645eb7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:58,849 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/.tmp/testFamily/8b284cc48de04637a845a7e0530b7b06 is 61, key is row-0/testFamily:testQualifier/1732071416647/Put/seqid=0 2024-11-20T02:57:58,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741866_1042 (size=5049) 2024-11-20T02:57:58,885 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:59,086 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:59,255 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/.tmp/testFamily/8b284cc48de04637a845a7e0530b7b06 2024-11-20T02:57:59,270 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/.tmp/testFamily/8b284cc48de04637a845a7e0530b7b06 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/testFamily/8b284cc48de04637a845a7e0530b7b06 2024-11-20T02:57:59,279 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/testFamily/8b284cc48de04637a845a7e0530b7b06, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:59,280 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 283fc959520ff14da243e54e3b645eb7 in 447ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:59,285 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/hbase/testMetaScanTimeoutOnOpenScanner/283fc959520ff14da243e54e3b645eb7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:59,286 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 283fc959520ff14da243e54e3b645eb7, 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:59,286 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:57:59,286 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 283fc959520ff14da243e54e3b645eb7: Waiting for close lock at 1732071478833Running coprocessor pre-close hooks at 1732071478833Disabling compacts and flushes for region at 1732071478833Disabling writes for close at 1732071478833Obtaining lock to block concurrent updates at 1732071478833Preparing flush snapshotting stores in 283fc959520ff14da243e54e3b645eb7 at 1732071478833Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071478834 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. at 1732071478834Flushing 283fc959520ff14da243e54e3b645eb7/testFamily: creating writer at 1732071478834Flushing 283fc959520ff14da243e54e3b645eb7/testFamily: appending metadata at 1732071478848 (+14 ms)Flushing 283fc959520ff14da243e54e3b645eb7/testFamily: closing flushed file at 1732071478848Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b1010d5: reopening flushed file at 1732071479269 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 283fc959520ff14da243e54e3b645eb7 in 447ms, sequenceid=8, compaction requested=false at 1732071479280 (+11 ms)Writing region close event to WAL at 1732071479281 (+1 ms)Running coprocessor post-close hooks at 1732071479286 (+5 ms)Closed at 1732071479286 2024-11-20T02:57:59,286 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732071413598.283fc959520ff14da243e54e3b645eb7. 2024-11-20T02:57:59,286 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55e149609e6ba0fd110650e3eeecaca2, disabling compactions & flushes 2024-11-20T02:57:59,287 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:57:59,287 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:57:59,287 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. after waiting 0 ms 2024-11-20T02:57:59,287 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:57:59,287 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 55e149609e6ba0fd110650e3eeecaca2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T02:57:59,302 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/.tmp/testFamily/51eb5078ee11438bb77648e6020f8fab is 61, key is row-0/testFamily:testQualifier/1732071413504/Put/seqid=0 2024-11-20T02:57:59,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741867_1043 (size=5049) 2024-11-20T02:57:59,486 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T02:57:59,486 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1325): Online Regions={55e149609e6ba0fd110650e3eeecaca2=testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2.} 2024-11-20T02:57:59,487 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:59,687 DEBUG [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1351): Waiting on 55e149609e6ba0fd110650e3eeecaca2 2024-11-20T02:57:59,710 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/.tmp/testFamily/51eb5078ee11438bb77648e6020f8fab 2024-11-20T02:57:59,723 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/.tmp/testFamily/51eb5078ee11438bb77648e6020f8fab as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/testFamily/51eb5078ee11438bb77648e6020f8fab 2024-11-20T02:57:59,732 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/testFamily/51eb5078ee11438bb77648e6020f8fab, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T02:57:59,733 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55e149609e6ba0fd110650e3eeecaca2 in 446ms, sequenceid=8, compaction requested=false 2024-11-20T02:57:59,739 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/55e149609e6ba0fd110650e3eeecaca2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T02:57:59,740 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:57:59,740 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55e149609e6ba0fd110650e3eeecaca2: Waiting for close lock at 1732071479286Running coprocessor pre-close hooks at 1732071479286Disabling compacts and flushes for region at 1732071479286Disabling writes for close at 1732071479287 (+1 ms)Obtaining lock to block concurrent updates at 1732071479287Preparing flush snapshotting stores in 55e149609e6ba0fd110650e3eeecaca2 at 1732071479287Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732071479287Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. at 1732071479288 (+1 ms)Flushing 55e149609e6ba0fd110650e3eeecaca2/testFamily: creating writer at 1732071479288Flushing 55e149609e6ba0fd110650e3eeecaca2/testFamily: appending metadata at 1732071479302 (+14 ms)Flushing 55e149609e6ba0fd110650e3eeecaca2/testFamily: closing flushed file at 1732071479302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3da153a1: reopening flushed file at 1732071479722 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55e149609e6ba0fd110650e3eeecaca2 in 446ms, sequenceid=8, compaction requested=false at 1732071479733 (+11 ms)Writing region close event to WAL at 1732071479734 (+1 ms)Running coprocessor post-close hooks at 1732071479740 (+6 ms)Closed at 1732071479740 2024-11-20T02:57:59,740 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732071410407.55e149609e6ba0fd110650e3eeecaca2. 2024-11-20T02:57:59,888 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,34099,1732071377033; all regions closed. 2024-11-20T02:57:59,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741834_1010 (size=19479) 2024-11-20T02:57:59,906 DEBUG [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/oldWALs 2024-11-20T02:57:59,906 INFO [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C34099%2C1732071377033.meta:.meta(num 1732071409929) 2024-11-20T02:57:59,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741833_1009 (size=20031) 2024-11-20T02:57:59,912 DEBUG [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/oldWALs 2024-11-20T02:57:59,912 INFO [RS:0;1c82b83631a2:34099 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C34099%2C1732071377033:(num 1732071409597) 2024-11-20T02:57:59,913 DEBUG [RS:0;1c82b83631a2:34099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:57:59,913 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:57:59,913 INFO [RS:0;1c82b83631a2:34099 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:57:59,913 INFO [RS:0;1c82b83631a2:34099 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T02:57:59,913 INFO [RS:0;1c82b83631a2:34099 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:57:59,913 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:57:59,915 INFO [RS:0;1c82b83631a2:34099 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34099 2024-11-20T02:57:59,937 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:57:59,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,34099,1732071377033 2024-11-20T02:57:59,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:57:59,979 INFO [RS:0;1c82b83631a2:34099 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:57:59,982 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,34099,1732071377033] 2024-11-20T02:57:59,999 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,34099,1732071377033 already deleted, retry=false 2024-11-20T02:58:00,000 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,34099,1732071377033 expired; onlineServers=0 2024-11-20T02:58:00,000 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1c82b83631a2,36265,1732071376244' ***** 2024-11-20T02:58:00,000 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T02:58:00,000 INFO [M:0;1c82b83631a2:36265 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:58:00,000 INFO [M:0;1c82b83631a2:36265 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:58:00,000 DEBUG [M:0;1c82b83631a2:36265 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T02:58:00,000 DEBUG [M:0;1c82b83631a2:36265 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T02:58:00,000 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T02:58:00,000 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071379332 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071379332,5,FailOnTimeoutGroup] 2024-11-20T02:58:00,000 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071379339 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071379339,5,FailOnTimeoutGroup] 2024-11-20T02:58:00,001 INFO [M:0;1c82b83631a2:36265 {}] hbase.ChoreService(370): Chore service for: master/1c82b83631a2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T02:58:00,001 INFO [M:0;1c82b83631a2:36265 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:58:00,001 DEBUG [M:0;1c82b83631a2:36265 {}] master.HMaster(1795): Stopping service threads 2024-11-20T02:58:00,001 INFO [M:0;1c82b83631a2:36265 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T02:58:00,001 INFO [M:0;1c82b83631a2:36265 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:58:00,002 INFO [M:0;1c82b83631a2:36265 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T02:58:00,002 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T02:58:00,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T02:58:00,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:58:00,012 DEBUG [M:0;1c82b83631a2:36265 {}] zookeeper.ZKUtil(347): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T02:58:00,012 WARN [M:0;1c82b83631a2:36265 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T02:58:00,013 INFO [M:0;1c82b83631a2:36265 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/.lastflushedseqids 2024-11-20T02:58:00,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741868_1044 (size=649) 2024-11-20T02:58:00,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:58:00,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34099-0x101577555050001, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:58:00,093 INFO [RS:0;1c82b83631a2:34099 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:58:00,093 INFO [RS:0;1c82b83631a2:34099 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,34099,1732071377033; zookeeper connection closed. 2024-11-20T02:58:00,094 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6912c82b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6912c82b 2024-11-20T02:58:00,095 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T02:58:00,429 INFO [M:0;1c82b83631a2:36265 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T02:58:00,430 INFO [M:0;1c82b83631a2:36265 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T02:58:00,430 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:58:00,431 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:58:00,431 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:58:00,431 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:58:00,431 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:58:00,432 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.07 KB heapSize=190.98 KB 2024-11-20T02:58:00,452 DEBUG [M:0;1c82b83631a2:36265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aece9bff34d8410ea28538013097bd00 is 82, key is hbase:meta,,1/info:regioninfo/1732071410024/Put/seqid=0 2024-11-20T02:58:00,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741869_1045 (size=5672) 2024-11-20T02:58:00,858 INFO [M:0;1c82b83631a2:36265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aece9bff34d8410ea28538013097bd00 2024-11-20T02:58:00,891 DEBUG [M:0;1c82b83631a2:36265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13c75ce79cc643b1bb7a98d34fee5278 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732071411686/Put/seqid=0 2024-11-20T02:58:00,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741870_1046 (size=14077) 2024-11-20T02:58:01,122 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:58:01,297 INFO [M:0;1c82b83631a2:36265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.52 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13c75ce79cc643b1bb7a98d34fee5278 2024-11-20T02:58:01,303 INFO [M:0;1c82b83631a2:36265 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 13c75ce79cc643b1bb7a98d34fee5278 2024-11-20T02:58:01,321 DEBUG [M:0;1c82b83631a2:36265 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/16574f9130424a44bdd86de2a4d641f5 is 69, key is 1c82b83631a2,34099,1732071377033/rs:state/1732071379398/Put/seqid=0 2024-11-20T02:58:01,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741871_1047 (size=5156) 2024-11-20T02:58:01,331 INFO [M:0;1c82b83631a2:36265 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/16574f9130424a44bdd86de2a4d641f5 2024-11-20T02:58:01,338 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aece9bff34d8410ea28538013097bd00 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aece9bff34d8410ea28538013097bd00 2024-11-20T02:58:01,345 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aece9bff34d8410ea28538013097bd00, entries=8, sequenceid=317, filesize=5.5 K 2024-11-20T02:58:01,346 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13c75ce79cc643b1bb7a98d34fee5278 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13c75ce79cc643b1bb7a98d34fee5278 2024-11-20T02:58:01,353 INFO [M:0;1c82b83631a2:36265 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 13c75ce79cc643b1bb7a98d34fee5278 2024-11-20T02:58:01,354 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13c75ce79cc643b1bb7a98d34fee5278, entries=33, sequenceid=317, filesize=13.7 K 2024-11-20T02:58:01,355 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/16574f9130424a44bdd86de2a4d641f5 as hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/16574f9130424a44bdd86de2a4d641f5 2024-11-20T02:58:01,362 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40175/user/jenkins/test-data/fe907a8f-8a2a-9e7c-7243-bbab58d5955b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/16574f9130424a44bdd86de2a4d641f5, entries=1, sequenceid=317, filesize=5.0 K 2024-11-20T02:58:01,363 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.07 KB/166987, heapSize ~190.68 KB/195256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 932ms, sequenceid=317, compaction requested=false 2024-11-20T02:58:01,366 INFO [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:58:01,366 DEBUG [M:0;1c82b83631a2:36265 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071480430Disabling compacts and flushes for region at 1732071480430Disabling writes for close at 1732071480431 (+1 ms)Obtaining lock to block concurrent updates at 1732071480432 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732071480432Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166987, getHeapSize=195496, getOffHeapSize=0, getCellsCount=364 at 1732071480433 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732071480435 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732071480436 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732071480452 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732071480452Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732071480869 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732071480890 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732071480890Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732071481303 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732071481320 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732071481320Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e15ee14: reopening flushed file at 1732071481337 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2006192a: reopening flushed file at 1732071481345 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d83c3e5: reopening flushed file at 1732071481354 (+9 ms)Finished flush of dataSize ~163.07 KB/166987, heapSize ~190.68 KB/195256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 932ms, sequenceid=317, compaction requested=false at 1732071481363 (+9 ms)Writing region close event to WAL at 1732071481366 (+3 ms)Closed at 1732071481366 2024-11-20T02:58:01,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37875 is added to blk_1073741830_1006 (size=188199) 2024-11-20T02:58:01,370 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:58:01,370 INFO [M:0;1c82b83631a2:36265 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T02:58:01,370 INFO [M:0;1c82b83631a2:36265 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36265 2024-11-20T02:58:01,370 INFO [M:0;1c82b83631a2:36265 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:58:01,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:58:01,519 INFO [M:0;1c82b83631a2:36265 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:58:01,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36265-0x101577555050000, quorum=127.0.0.1:57317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:58:01,550 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T02:58:01,552 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T02:58:01,552 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T02:58:01,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T02:58:01,552 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.log.dir/,STOPPED} 2024-11-20T02:58:01,555 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T02:58:01,555 WARN [BP-2052504977-172.17.0.2-1732071372787 heartbeating to localhost/127.0.0.1:40175 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T02:58:01,555 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T02:58:01,555 WARN [BP-2052504977-172.17.0.2-1732071372787 heartbeating to localhost/127.0.0.1:40175 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2052504977-172.17.0.2-1732071372787 (Datanode Uuid 69e1f7cc-78d0-4659-abb8-aeaa876f66db) service to localhost/127.0.0.1:40175 2024-11-20T02:58:01,557 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/data/data1/current/BP-2052504977-172.17.0.2-1732071372787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T02:58:01,557 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/cluster_4e5a27e3-7e4e-c27f-3e5d-189e601ce608/data/data2/current/BP-2052504977-172.17.0.2-1732071372787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T02:58:01,557 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T02:58:01,596 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T02:58:01,597 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T02:58:01,597 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T02:58:01,597 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T02:58:01,598 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0ede4b59-d655-d712-c9de-69d26fc1d1b4/hadoop.log.dir/,STOPPED} 2024-11-20T02:58:01,607 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T02:58:01,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down