2024-12-05 20:00:30,631 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-05 20:00:30,646 main DEBUG Took 0.013150 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 20:00:30,647 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 20:00:30,647 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 20:00:30,648 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 20:00:30,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,676 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 20:00:30,692 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,696 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,697 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,697 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,698 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,700 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,701 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,702 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,702 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,703 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,704 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,705 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,705 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 20:00:30,706 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,707 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 20:00:30,709 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 20:00:30,710 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 20:00:30,712 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 20:00:30,713 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 20:00:30,714 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 20:00:30,715 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 20:00:30,725 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 20:00:30,729 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 20:00:30,730 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 20:00:30,731 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 20:00:30,731 main DEBUG createAppenders(={Console}) 2024-12-05 20:00:30,732 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-05 20:00:30,733 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-05 20:00:30,733 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-05 20:00:30,734 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 20:00:30,734 main DEBUG OutputStream closed 2024-12-05 20:00:30,734 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 20:00:30,735 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 20:00:30,735 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-05 20:00:30,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 20:00:30,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 20:00:30,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 20:00:30,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 20:00:30,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 20:00:30,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 20:00:30,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 20:00:30,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 20:00:30,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 20:00:30,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 20:00:30,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 20:00:30,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 20:00:30,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 20:00:30,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 20:00:30,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 20:00:30,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 20:00:30,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 20:00:30,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 20:00:30,833 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 20:00:30,834 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-05 20:00:30,834 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 20:00:30,835 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-05T20:00:30,848 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-05 20:00:30,851 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 20:00:30,851 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T20:00:31,162 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68 2024-12-05T20:00:31,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T20:00:31,208 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e, deleteOnExit=true 2024-12-05T20:00:31,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T20:00:31,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/test.cache.data in system properties and HBase conf 2024-12-05T20:00:31,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T20:00:31,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.log.dir in system properties and HBase conf 2024-12-05T20:00:31,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T20:00:31,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T20:00:31,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T20:00:31,328 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T20:00:31,442 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T20:00:31,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T20:00:31,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T20:00:31,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T20:00:31,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T20:00:31,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T20:00:31,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T20:00:31,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T20:00:31,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T20:00:31,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T20:00:31,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/nfs.dump.dir in system properties and HBase conf 2024-12-05T20:00:31,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/java.io.tmpdir in system properties and HBase conf 2024-12-05T20:00:31,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T20:00:31,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T20:00:31,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T20:00:32,427 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T20:00:32,517 INFO [Time-limited test {}] log.Log(170): Logging initialized @2798ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T20:00:32,615 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T20:00:32,681 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T20:00:32,702 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T20:00:32,702 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T20:00:32,703 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T20:00:32,717 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T20:00:32,720 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.log.dir/,AVAILABLE} 2024-12-05T20:00:32,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T20:00:32,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/java.io.tmpdir/jetty-localhost-33071-hadoop-hdfs-3_4_1-tests_jar-_-any-5206140490889953368/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T20:00:32,967 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:33071} 2024-12-05T20:00:32,968 INFO [Time-limited test {}] server.Server(415): Started @3250ms 2024-12-05T20:00:33,421 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T20:00:33,431 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T20:00:33,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T20:00:33,436 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T20:00:33,437 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T20:00:33,438 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.log.dir/,AVAILABLE} 2024-12-05T20:00:33,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T20:00:33,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/java.io.tmpdir/jetty-localhost-42345-hadoop-hdfs-3_4_1-tests_jar-_-any-15993761306282445544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T20:00:33,579 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:42345} 2024-12-05T20:00:33,579 INFO [Time-limited test {}] server.Server(415): Started @3862ms 2024-12-05T20:00:33,647 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T20:00:34,253 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/data/data2/current/BP-1670072369-172.17.0.2-1733428832170/current, will proceed with Du for space computation calculation, 2024-12-05T20:00:34,253 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/data/data1/current/BP-1670072369-172.17.0.2-1733428832170/current, will proceed with Du for space computation calculation, 2024-12-05T20:00:34,309 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T20:00:34,366 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x488d80935b5583a4 with lease ID 0xd189b3a4a671d8c1: Processing first storage report for DS-14032d1f-aa4a-40a8-a0a8-08e3fb3f0672 from datanode DatanodeRegistration(127.0.0.1:40253, datanodeUuid=00a7e687-701c-469a-b475-6989e42e972e, infoPort=38011, infoSecurePort=0, ipcPort=34589, storageInfo=lv=-57;cid=testClusterID;nsid=2007579264;c=1733428832170) 2024-12-05T20:00:34,368 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x488d80935b5583a4 with lease ID 0xd189b3a4a671d8c1: from storage DS-14032d1f-aa4a-40a8-a0a8-08e3fb3f0672 node DatanodeRegistration(127.0.0.1:40253, datanodeUuid=00a7e687-701c-469a-b475-6989e42e972e, infoPort=38011, infoSecurePort=0, ipcPort=34589, storageInfo=lv=-57;cid=testClusterID;nsid=2007579264;c=1733428832170), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T20:00:34,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x488d80935b5583a4 with lease ID 0xd189b3a4a671d8c1: Processing first storage report for DS-b4bab3dc-bd64-4e69-8850-0e2a19045248 from datanode DatanodeRegistration(127.0.0.1:40253, datanodeUuid=00a7e687-701c-469a-b475-6989e42e972e, infoPort=38011, infoSecurePort=0, ipcPort=34589, storageInfo=lv=-57;cid=testClusterID;nsid=2007579264;c=1733428832170) 2024-12-05T20:00:34,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x488d80935b5583a4 with lease ID 0xd189b3a4a671d8c1: from storage DS-b4bab3dc-bd64-4e69-8850-0e2a19045248 node DatanodeRegistration(127.0.0.1:40253, datanodeUuid=00a7e687-701c-469a-b475-6989e42e972e, infoPort=38011, infoSecurePort=0, ipcPort=34589, storageInfo=lv=-57;cid=testClusterID;nsid=2007579264;c=1733428832170), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T20:00:34,440 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68 2024-12-05T20:00:34,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/zookeeper_0, clientPort=51307, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T20:00:34,547 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51307 2024-12-05T20:00:34,561 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:34,565 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:34,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741825_1001 (size=7) 2024-12-05T20:00:35,238 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc with version=8 2024-12-05T20:00:35,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/hbase-staging 2024-12-05T20:00:35,359 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T20:00:35,640 INFO [Time-limited test {}] client.ConnectionUtils(128): master/86162e2766a8:0 server-side Connection retries=9 2024-12-05T20:00:35,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:35,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:35,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T20:00:35,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:35,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T20:00:35,866 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T20:00:35,940 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T20:00:35,952 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T20:00:35,957 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T20:00:35,988 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14545 (auto-detected) 2024-12-05T20:00:35,989 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T20:00:36,010 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41067 2024-12-05T20:00:36,039 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41067 connecting to ZooKeeper ensemble=127.0.0.1:51307 2024-12-05T20:00:36,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:410670x0, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T20:00:36,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41067-0x10063c4a9b50000 connected 2024-12-05T20:00:36,124 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:36,127 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:36,139 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T20:00:36,144 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc, hbase.cluster.distributed=false 2024-12-05T20:00:36,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T20:00:36,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41067 2024-12-05T20:00:36,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41067 2024-12-05T20:00:36,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41067 2024-12-05T20:00:36,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41067 2024-12-05T20:00:36,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41067 2024-12-05T20:00:36,350 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/86162e2766a8:0 server-side Connection retries=9 2024-12-05T20:00:36,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:36,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:36,353 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T20:00:36,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T20:00:36,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T20:00:36,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T20:00:36,358 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T20:00:36,359 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39051 2024-12-05T20:00:36,361 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39051 connecting to ZooKeeper ensemble=127.0.0.1:51307 2024-12-05T20:00:36,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:36,367 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:36,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390510x0, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T20:00:36,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39051-0x10063c4a9b50001 connected 2024-12-05T20:00:36,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T20:00:36,381 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T20:00:36,390 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T20:00:36,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T20:00:36,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T20:00:36,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39051 2024-12-05T20:00:36,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39051 2024-12-05T20:00:36,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39051 2024-12-05T20:00:36,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39051 2024-12-05T20:00:36,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39051 2024-12-05T20:00:36,424 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;86162e2766a8:41067 2024-12-05T20:00:36,425 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/86162e2766a8,41067,1733428835413 2024-12-05T20:00:36,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T20:00:36,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T20:00:36,436 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/86162e2766a8,41067,1733428835413 2024-12-05T20:00:36,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T20:00:36,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:36,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:36,459 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T20:00:36,461 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/86162e2766a8,41067,1733428835413 from backup master directory 2024-12-05T20:00:36,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/86162e2766a8,41067,1733428835413 2024-12-05T20:00:36,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T20:00:36,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T20:00:36,465 WARN [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T20:00:36,465 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=86162e2766a8,41067,1733428835413 2024-12-05T20:00:36,468 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T20:00:36,471 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T20:00:36,539 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/hbase.id] with ID: b552ce22-044b-4ab1-8792-3f2618b9ec90 2024-12-05T20:00:36,539 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/.tmp/hbase.id 2024-12-05T20:00:36,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741826_1002 (size=42) 2024-12-05T20:00:36,554 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/.tmp/hbase.id]:[hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/hbase.id] 2024-12-05T20:00:36,604 INFO [master/86162e2766a8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T20:00:36,611 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T20:00:36,638 INFO [master/86162e2766a8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-12-05T20:00:36,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:36,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:36,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741827_1003 (size=196) 2024-12-05T20:00:36,680 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:00:36,684 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T20:00:36,703 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T20:00:36,709 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T20:00:36,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741828_1004 (size=1189) 2024-12-05T20:00:37,168 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store 2024-12-05T20:00:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741829_1005 (size=34) 2024-12-05T20:00:37,599 INFO [master/86162e2766a8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T20:00:37,602 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:00:37,603 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T20:00:37,604 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:00:37,604 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:00:37,605 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T20:00:37,606 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:00:37,606 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:00:37,607 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733428837603Disabling compacts and flushes for region at 1733428837603Disabling writes for close at 1733428837605 (+2 ms)Writing region close event to WAL at 1733428837606 (+1 ms)Closed at 1733428837606 2024-12-05T20:00:37,609 WARN [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/.initializing 2024-12-05T20:00:37,609 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/WALs/86162e2766a8,41067,1733428835413 2024-12-05T20:00:37,618 INFO [master/86162e2766a8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T20:00:37,634 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=86162e2766a8%2C41067%2C1733428835413, suffix=, logDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/WALs/86162e2766a8,41067,1733428835413, archiveDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/oldWALs, maxLogs=10 2024-12-05T20:00:37,658 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/WALs/86162e2766a8,41067,1733428835413/86162e2766a8%2C41067%2C1733428835413.1733428837640, exclude list is [], retry=0 2024-12-05T20:00:37,678 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-14032d1f-aa4a-40a8-a0a8-08e3fb3f0672,DISK] 2024-12-05T20:00:37,682 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T20:00:37,728 INFO [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/WALs/86162e2766a8,41067,1733428835413/86162e2766a8%2C41067%2C1733428835413.1733428837640 2024-12-05T20:00:37,729 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38011:38011)] 2024-12-05T20:00:37,730 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:00:37,730 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:00:37,734 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,735 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T20:00:37,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:37,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:00:37,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T20:00:37,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:37,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:00:37,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T20:00:37,833 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:37,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:00:37,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T20:00:37,838 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:37,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:00:37,839 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,843 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,844 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,850 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,851 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,855 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T20:00:37,859 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T20:00:37,864 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:00:37,866 INFO [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74707439, jitterRate=0.11322759091854095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T20:00:37,873 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733428837747Initializing all the Stores at 1733428837749 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428837750 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428837750Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428837751 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428837751Cleaning up temporary data from old regions at 1733428837851 (+100 ms)Region opened successfully at 1733428837873 (+22 ms) 2024-12-05T20:00:37,875 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T20:00:37,916 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f2e21fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T20:00:37,953 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T20:00:37,966 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T20:00:37,966 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T20:00:37,970 INFO [master/86162e2766a8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T20:00:37,971 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T20:00:37,976 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T20:00:37,977 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T20:00:38,005 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T20:00:38,016 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T20:00:38,018 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T20:00:38,022 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T20:00:38,024 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T20:00:38,025 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T20:00:38,028 INFO [master/86162e2766a8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T20:00:38,033 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T20:00:38,034 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T20:00:38,036 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T20:00:38,038 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T20:00:38,061 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T20:00:38,063 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T20:00:38,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T20:00:38,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T20:00:38,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,070 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=86162e2766a8,41067,1733428835413, sessionid=0x10063c4a9b50000, setting cluster-up flag (Was=false) 2024-12-05T20:00:38,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,091 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T20:00:38,093 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,41067,1733428835413 2024-12-05T20:00:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:00:38,106 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T20:00:38,108 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=86162e2766a8,41067,1733428835413 2024-12-05T20:00:38,115 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T20:00:38,205 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T20:00:38,212 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(746): ClusterId : b552ce22-044b-4ab1-8792-3f2618b9ec90 2024-12-05T20:00:38,215 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T20:00:38,217 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T20:00:38,219 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T20:00:38,220 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T20:00:38,223 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T20:00:38,224 INFO [master/86162e2766a8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T20:00:38,224 DEBUG [RS:0;86162e2766a8:39051 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669b728e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=86162e2766a8/172.17.0.2:0 2024-12-05T20:00:38,230 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 86162e2766a8,41067,1733428835413 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T20:00:38,237 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T20:00:38,237 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T20:00:38,237 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T20:00:38,238 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/86162e2766a8:0, corePoolSize=5, maxPoolSize=5 2024-12-05T20:00:38,238 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/86162e2766a8:0, corePoolSize=10, maxPoolSize=10 2024-12-05T20:00:38,238 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,238 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T20:00:38,239 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,244 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;86162e2766a8:39051 2024-12-05T20:00:38,246 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733428868246 2024-12-05T20:00:38,248 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T20:00:38,248 INFO [RS:0;86162e2766a8:39051 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T20:00:38,248 INFO [RS:0;86162e2766a8:39051 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T20:00:38,249 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T20:00:38,249 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T20:00:38,250 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T20:00:38,250 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T20:00:38,252 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(2659): reportForDuty to master=86162e2766a8,41067,1733428835413 with port=39051, startcode=1733428836300 2024-12-05T20:00:38,254 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T20:00:38,255 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T20:00:38,255 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T20:00:38,256 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T20:00:38,257 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:38,258 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T20:00:38,259 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,268 DEBUG [RS:0;86162e2766a8:39051 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T20:00:38,268 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T20:00:38,269 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T20:00:38,270 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T20:00:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741831_1007 (size=1321) 2024-12-05T20:00:38,276 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T20:00:38,277 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:00:38,284 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T20:00:38,284 INFO [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T20:00:38,291 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733428838286,5,FailOnTimeoutGroup] 2024-12-05T20:00:38,295 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733428838291,5,FailOnTimeoutGroup] 2024-12-05T20:00:38,296 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,296 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T20:00:38,297 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,298 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741832_1008 (size=32) 2024-12-05T20:00:38,314 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T20:00:38,320 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41067 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,322 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41067 {}] master.ServerManager(517): Registering regionserver=86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,337 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:00:38,337 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41001 2024-12-05T20:00:38,337 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T20:00:38,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T20:00:38,342 DEBUG [RS:0;86162e2766a8:39051 {}] zookeeper.ZKUtil(111): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,343 WARN [RS:0;86162e2766a8:39051 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T20:00:38,343 INFO [RS:0;86162e2766a8:39051 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T20:00:38,343 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,347 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [86162e2766a8,39051,1733428836300] 2024-12-05T20:00:38,371 INFO [RS:0;86162e2766a8:39051 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T20:00:38,388 INFO [RS:0;86162e2766a8:39051 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T20:00:38,395 INFO [RS:0;86162e2766a8:39051 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T20:00:38,395 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,396 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T20:00:38,403 INFO [RS:0;86162e2766a8:39051 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T20:00:38,404 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,405 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,405 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,405 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,405 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,405 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/86162e2766a8:0, corePoolSize=2, maxPoolSize=2 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,406 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/86162e2766a8:0, corePoolSize=1, maxPoolSize=1 2024-12-05T20:00:38,407 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T20:00:38,407 DEBUG [RS:0;86162e2766a8:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/86162e2766a8:0, corePoolSize=3, maxPoolSize=3 2024-12-05T20:00:38,408 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,408 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,408 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,409 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,409 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,409 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,39051,1733428836300-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T20:00:38,428 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T20:00:38,430 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,39051,1733428836300-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,430 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,431 INFO [RS:0;86162e2766a8:39051 {}] regionserver.Replication(171): 86162e2766a8,39051,1733428836300 started 2024-12-05T20:00:38,455 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:00:38,455 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1482): Serving as 86162e2766a8,39051,1733428836300, RpcServer on 86162e2766a8/172.17.0.2:39051, sessionid=0x10063c4a9b50001 2024-12-05T20:00:38,456 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T20:00:38,456 DEBUG [RS:0;86162e2766a8:39051 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,457 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,39051,1733428836300' 2024-12-05T20:00:38,457 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T20:00:38,458 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T20:00:38,459 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T20:00:38,459 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T20:00:38,459 DEBUG [RS:0;86162e2766a8:39051 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 86162e2766a8,39051,1733428836300 2024-12-05T20:00:38,459 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '86162e2766a8,39051,1733428836300' 2024-12-05T20:00:38,459 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T20:00:38,460 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T20:00:38,460 DEBUG [RS:0;86162e2766a8:39051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T20:00:38,460 INFO [RS:0;86162e2766a8:39051 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T20:00:38,461 INFO [RS:0;86162e2766a8:39051 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T20:00:38,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:00:38,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T20:00:38,716 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T20:00:38,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:38,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:00:38,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T20:00:38,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T20:00:38,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:38,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:00:38,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T20:00:38,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T20:00:38,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:38,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:00:38,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T20:00:38,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T20:00:38,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:00:38,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:00:38,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T20:00:38,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740 2024-12-05T20:00:38,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740 2024-12-05T20:00:38,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T20:00:38,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T20:00:38,736 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T20:00:38,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T20:00:38,743 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:00:38,744 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68280096, jitterRate=0.017452716827392578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T20:00:38,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733428838710Initializing all the Stores at 1733428838713 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428838713Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428838713Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428838713Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428838713Cleaning up temporary data from old regions at 1733428838735 (+22 ms)Region opened successfully at 1733428838747 (+12 ms) 2024-12-05T20:00:38,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T20:00:38,747 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T20:00:38,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T20:00:38,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T20:00:38,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T20:00:38,749 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T20:00:38,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733428838747Disabling compacts and flushes for region at 1733428838747Disabling writes for close at 1733428838748 (+1 ms)Writing region close event to WAL at 1733428838749 (+1 ms)Closed at 1733428838749 2024-12-05T20:00:38,753 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T20:00:38,753 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T20:00:38,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T20:00:38,769 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T20:00:38,772 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T20:00:38,923 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:39,925 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:40,925 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:41,926 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:42,927 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:43,927 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:44,928 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:45,928 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:45,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T20:00:45,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T20:00:46,929 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:47,930 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:48,931 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:49,931 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:50,932 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:51,933 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:52,933 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:53,934 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:54,935 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:55,935 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:56,936 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:57,940 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:58,941 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:00:59,943 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:00,947 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:01,948 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:02,949 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:03,950 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:04,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T20:01:04,951 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:05,952 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:06,956 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:07,958 WARN [86162e2766a8:41067 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T20:01:08,470 INFO [RS:0;86162e2766a8:39051 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T20:01:08,519 INFO [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=86162e2766a8%2C39051%2C1733428836300, suffix=, logDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300, archiveDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/oldWALs, maxLogs=32 2024-12-05T20:01:08,544 DEBUG [RS:0;86162e2766a8:39051 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300/86162e2766a8%2C39051%2C1733428836300.1733428868523, exclude list is [], retry=0 2024-12-05T20:01:08,566 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-14032d1f-aa4a-40a8-a0a8-08e3fb3f0672,DISK] 2024-12-05T20:01:08,616 INFO [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300/86162e2766a8%2C39051%2C1733428836300.1733428868523 2024-12-05T20:01:08,651 DEBUG [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38011:38011)] 2024-12-05T20:01:08,715 DEBUG [86162e2766a8:41067 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T20:01:08,754 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:08,773 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,39051,1733428836300, state=OPENING 2024-12-05T20:01:08,796 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T20:01:08,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:01:08,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:01:08,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T20:01:08,800 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T20:01:08,804 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T20:01:08,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:09,033 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T20:01:09,077 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T20:01:09,181 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T20:01:09,182 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T20:01:09,197 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T20:01:09,213 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=86162e2766a8%2C39051%2C1733428836300.meta, suffix=.meta, logDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300, archiveDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/oldWALs, maxLogs=32 2024-12-05T20:01:09,237 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300/86162e2766a8%2C39051%2C1733428836300.meta.1733428869215.meta, exclude list is [], retry=0 2024-12-05T20:01:09,265 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-14032d1f-aa4a-40a8-a0a8-08e3fb3f0672,DISK] 2024-12-05T20:01:09,288 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300/86162e2766a8%2C39051%2C1733428836300.meta.1733428869215.meta 2024-12-05T20:01:09,291 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38011:38011)] 2024-12-05T20:01:09,291 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:09,294 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T20:01:09,298 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T20:01:09,327 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T20:01:09,333 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T20:01:09,334 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:09,335 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T20:01:09,335 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T20:01:09,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T20:01:09,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T20:01:09,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:09,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:01:09,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T20:01:09,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T20:01:09,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:09,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:01:09,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T20:01:09,457 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T20:01:09,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:09,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:01:09,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T20:01:09,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T20:01:09,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:09,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T20:01:09,499 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T20:01:09,505 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740 2024-12-05T20:01:09,513 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740 2024-12-05T20:01:09,526 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T20:01:09,526 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T20:01:09,528 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T20:01:09,531 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T20:01:09,533 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74925399, jitterRate=0.11647544801235199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T20:01:09,534 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T20:01:09,536 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733428869336Writing region info on filesystem at 1733428869336Initializing all the Stores at 1733428869370 (+34 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428869370Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428869375 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428869376 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733428869376Cleaning up temporary data from old regions at 1733428869526 (+150 ms)Running coprocessor post-open hooks at 1733428869534 (+8 ms)Region opened successfully at 1733428869536 (+2 ms) 2024-12-05T20:01:09,563 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733428869016 2024-12-05T20:01:09,593 DEBUG [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T20:01:09,595 INFO [RS_OPEN_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T20:01:09,597 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:09,604 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 86162e2766a8,39051,1733428836300, state=OPEN 2024-12-05T20:01:09,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T20:01:09,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T20:01:09,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T20:01:09,625 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T20:01:09,626 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=86162e2766a8,39051,1733428836300 2024-12-05T20:01:09,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T20:01:09,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=86162e2766a8,39051,1733428836300 in 820 msec 2024-12-05T20:01:09,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T20:01:09,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.8850 sec 2024-12-05T20:01:09,755 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T20:01:09,755 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T20:01:09,825 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T20:01:09,835 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,39051,1733428836300, seqNum=-1] 2024-12-05T20:01:10,093 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T20:01:10,097 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36635, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T20:01:10,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 32.0370 sec 2024-12-05T20:01:10,216 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733428870215, completionTime=-1 2024-12-05T20:01:10,231 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T20:01:10,231 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T20:01:10,287 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T20:01:10,287 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733428930287 2024-12-05T20:01:10,287 INFO [master/86162e2766a8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733428990287 2024-12-05T20:01:10,287 INFO [master/86162e2766a8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 55 msec 2024-12-05T20:01:10,291 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,291 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,293 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,295 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-86162e2766a8:41067, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,303 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,310 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,313 DEBUG [master/86162e2766a8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T20:01:10,373 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.908sec 2024-12-05T20:01:10,375 INFO [master/86162e2766a8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T20:01:10,377 INFO [master/86162e2766a8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T20:01:10,381 INFO [master/86162e2766a8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T20:01:10,382 INFO [master/86162e2766a8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T20:01:10,382 INFO [master/86162e2766a8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T20:01:10,383 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T20:01:10,384 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T20:01:10,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:10,467 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T20:01:10,467 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T20:01:10,477 DEBUG [master/86162e2766a8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T20:01:10,478 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T20:01:10,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 86162e2766a8,41067,-1 for getting cluster id 2024-12-05T20:01:10,479 INFO [master/86162e2766a8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=86162e2766a8,41067,1733428835413-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T20:01:10,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T20:01:10,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b552ce22-044b-4ab1-8792-3f2618b9ec90' 2024-12-05T20:01:10,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T20:01:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b552ce22-044b-4ab1-8792-3f2618b9ec90" 2024-12-05T20:01:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b758490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [86162e2766a8,41067,-1] 2024-12-05T20:01:10,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T20:01:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:01:10,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T20:01:10,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:10,730 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T20:01:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,39051,1733428836300, seqNum=-1] 2024-12-05T20:01:10,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T20:01:10,806 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T20:01:10,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=86162e2766a8,41067,1733428835413 2024-12-05T20:01:10,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd93155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:10,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 86162e2766a8,41067,-1 for getting cluster id 2024-12-05T20:01:10,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T20:01:10,894 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b552ce22-044b-4ab1-8792-3f2618b9ec90' 2024-12-05T20:01:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T20:01:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b552ce22-044b-4ab1-8792-3f2618b9ec90" 2024-12-05T20:01:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd4d5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [86162e2766a8,41067,-1] 2024-12-05T20:01:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T20:01:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:01:10,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T20:01:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467ca67d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T20:01:11,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1165, ProcessCount=11, AvailableMemoryMB=9862 2024-12-05T20:01:11,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T20:01:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 86162e2766a8,41067,1733428835413 2024-12-05T20:01:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d3d2ccb 2024-12-05T20:01:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T20:01:11,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T20:01:11,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-05T20:01:11,141 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:11,144 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:11,164 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:11,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-05T20:01:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T20:01:11,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741835_1011 (size=428) 2024-12-05T20:01:11,797 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9079d4b3f85f7c7f362b6f05e31b5422, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:11,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741836_1012 (size=79) 2024-12-05T20:01:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T20:01:12,353 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:12,354 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9079d4b3f85f7c7f362b6f05e31b5422, disabling compactions & flushes 2024-12-05T20:01:12,354 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:12,354 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:12,354 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. after waiting 0 ms 2024-12-05T20:01:12,354 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:12,354 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:12,354 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9079d4b3f85f7c7f362b6f05e31b5422: Waiting for close lock at 1733428872353Disabling compacts and flushes for region at 1733428872353Disabling writes for close at 1733428872354 (+1 ms)Writing region close event to WAL at 1733428872354Closed at 1733428872354 2024-12-05T20:01:12,362 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:12,369 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733428872362"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428872362"}]},"ts":"1733428872362"} 2024-12-05T20:01:12,390 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:12,392 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:12,396 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428872393"}]},"ts":"1733428872393"} 2024-12-05T20:01:12,410 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-05T20:01:12,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9079d4b3f85f7c7f362b6f05e31b5422, ASSIGN}] 2024-12-05T20:01:12,427 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9079d4b3f85f7c7f362b6f05e31b5422, ASSIGN 2024-12-05T20:01:12,429 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9079d4b3f85f7c7f362b6f05e31b5422, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:12,587 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9079d4b3f85f7c7f362b6f05e31b5422, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:12,599 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9079d4b3f85f7c7f362b6f05e31b5422, ASSIGN because future has completed 2024-12-05T20:01:12,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9079d4b3f85f7c7f362b6f05e31b5422, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:12,860 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:12,861 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9079d4b3f85f7c7f362b6f05e31b5422, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:12,862 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,862 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:12,862 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,880 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,909 INFO [StoreOpener-9079d4b3f85f7c7f362b6f05e31b5422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,932 INFO [StoreOpener-9079d4b3f85f7c7f362b6f05e31b5422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9079d4b3f85f7c7f362b6f05e31b5422 columnFamilyName testFamily 2024-12-05T20:01:12,933 DEBUG [StoreOpener-9079d4b3f85f7c7f362b6f05e31b5422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:12,950 INFO [StoreOpener-9079d4b3f85f7c7f362b6f05e31b5422-1 {}] regionserver.HStore(327): Store=9079d4b3f85f7c7f362b6f05e31b5422/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:12,972 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,974 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,981 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,982 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,982 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:12,989 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:13,028 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:13,037 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9079d4b3f85f7c7f362b6f05e31b5422; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60323837, jitterRate=-0.10110478103160858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:13,037 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:01:13,038 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9079d4b3f85f7c7f362b6f05e31b5422: Running coprocessor pre-open hook at 1733428872880Writing region info on filesystem at 1733428872880Initializing all the Stores at 1733428872892 (+12 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428872892Cleaning up temporary data from old regions at 1733428872982 (+90 ms)Running coprocessor post-open hooks at 1733428873037 (+55 ms)Region opened successfully at 1733428873038 (+1 ms) 2024-12-05T20:01:13,043 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422., pid=6, masterSystemTime=1733428872832 2024-12-05T20:01:13,049 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:13,049 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:01:13,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9079d4b3f85f7c7f362b6f05e31b5422, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:13,157 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9079d4b3f85f7c7f362b6f05e31b5422, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:13,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T20:01:13,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9079d4b3f85f7c7f362b6f05e31b5422, server=86162e2766a8,39051,1733428836300 in 495 msec 2024-12-05T20:01:13,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T20:01:13,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=9079d4b3f85f7c7f362b6f05e31b5422, ASSIGN in 795 msec 2024-12-05T20:01:13,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:13,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428873233"}]},"ts":"1733428873233"} 2024-12-05T20:01:13,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-05T20:01:13,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:13,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 2.1550 sec 2024-12-05T20:01:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T20:01:14,299 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-05T20:01:14,300 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-05T20:01:14,304 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:14,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-05T20:01:14,321 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:14,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-05T20:01:14,327 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T20:01:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=86162e2766a8,39051,1733428836300, seqNum=-1] 2024-12-05T20:01:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T20:01:14,400 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T20:01:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:14,510 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:14,515 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-05T20:01:14,522 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-05T20:01:14,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-05T20:01:14,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7374406577480007672 2024-12-05T20:01:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:55550 deadline: 1733428877552, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T20:01:14,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=219 (was 214) Potentially hanging thread: RS_OPEN_REGION-regionserver/86162e2766a8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/java.util.zip.ZipFile$Source.getEntryPos(ZipFile.java:1809) java.base@17.0.11/java.util.zip.ZipFile.getEntry(ZipFile.java:340) java.base@17.0.11/java.util.jar.JarFile.getEntry(JarFile.java:517) java.base@17.0.11/java.util.jar.JarFile.getJarEntry(JarFile.java:472) java.base@17.0.11/jdk.internal.loader.URLClassPath$JarLoader.getResource(URLClassPath.java:954) java.base@17.0.11/jdk.internal.loader.URLClassPath$JarLoader.findResource(URLClassPath.java:937) java.base@17.0.11/jdk.internal.loader.URLClassPath.findResource(URLClassPath.java:296) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.lambda$findResourceOnClassPath$0(BuiltinClassLoader.java:539) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$$Lambda$142/0x00007f0a40161818.run(Unknown Source) java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findResourceOnClassPath(BuiltinClassLoader.java:540) java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findResource(BuiltinClassLoader.java:355) java.base@17.0.11/java.lang.ClassLoader.getResource(ClassLoader.java:1408) app//org.apache.hadoop.conf.Configuration.getResource(Configuration.java:2883) app//org.apache.hadoop.conf.Configuration.getStreamReader(Configuration.java:3157) app//org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:3116) app//org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:3089) app//org.apache.hadoop.conf.Configuration.loadProps(Configuration.java:2967) app//org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2949) app//org.apache.hadoop.conf.Configuration.get(Configuration.java:1267) app//org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:1321) app//org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1728) app//org.apache.hadoop.hbase.HBaseConfiguration.checkDefaultsVersion(HBaseConfiguration.java:42) app//org.apache.hadoop.hbase.HBaseConfiguration.addHbaseResources(HBaseConfiguration.java:91) app//org.apache.hadoop.hbase.HBaseConfiguration.create(HBaseConfiguration.java:105) app//org.apache.hadoop.hbase.ipc.RemoteWithExtrasException$ClassLoaderHolder.(RemoteWithExtrasException.java:54) app//org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:89) app//org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) app//org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller.onError(AsyncScanSingleRegionRpcRetryingCaller.java:415) app//org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller.onComplete(AsyncScanSingleRegionRpcRetryingCaller.java:507) app//org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller.lambda$call$3(AsyncScanSingleRegionRpcRetryingCaller.java:604) app//org.apache.hadoop.hbase.client.AsyncScanSingleRegionRpcRetryingCaller$$Lambda$900/0x00007f0a40bf3268.run(Unknown Source) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) app//org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) app//org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) app//org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) app//org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) app//org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) app//org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-167491816_22 at /127.0.0.1:44432 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1165 (was 1165), ProcessCount=11 (was 11), AvailableMemoryMB=9825 (was 9862) 2024-12-05T20:01:14,927 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:14,978 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=219, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1165, ProcessCount=11, AvailableMemoryMB=9821 2024-12-05T20:01:14,997 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:15,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-05T20:01:15,003 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:15,003 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:15,005 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-05T20:01:15,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T20:01:15,072 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-05T20:01:15,075 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T20:01:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741837_1013 (size=414) 2024-12-05T20:01:15,225 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb3adcb22c2a13d801d409bcb84d1d55, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741838_1014 (size=65) 2024-12-05T20:01:15,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:15,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing eb3adcb22c2a13d801d409bcb84d1d55, disabling compactions & flushes 2024-12-05T20:01:15,430 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:15,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:15,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. after waiting 0 ms 2024-12-05T20:01:15,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:15,431 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:15,431 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb3adcb22c2a13d801d409bcb84d1d55: Waiting for close lock at 1733428875430Disabling compacts and flushes for region at 1733428875430Disabling writes for close at 1733428875430Writing region close event to WAL at 1733428875430Closed at 1733428875430 2024-12-05T20:01:15,435 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:15,436 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733428875435"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428875435"}]},"ts":"1733428875435"} 2024-12-05T20:01:15,460 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:15,468 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:15,468 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428875468"}]},"ts":"1733428875468"} 2024-12-05T20:01:15,490 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-05T20:01:15,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=eb3adcb22c2a13d801d409bcb84d1d55, ASSIGN}] 2024-12-05T20:01:15,494 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=eb3adcb22c2a13d801d409bcb84d1d55, ASSIGN 2024-12-05T20:01:15,499 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=eb3adcb22c2a13d801d409bcb84d1d55, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:15,651 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eb3adcb22c2a13d801d409bcb84d1d55, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:15,677 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=eb3adcb22c2a13d801d409bcb84d1d55, ASSIGN because future has completed 2024-12-05T20:01:15,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb3adcb22c2a13d801d409bcb84d1d55, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:15,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T20:01:15,948 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:15,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T20:01:15,953 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T20:01:15,954 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-05T20:01:15,954 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T20:01:16,115 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:16,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => eb3adcb22c2a13d801d409bcb84d1d55, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:16,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:16,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,275 INFO [StoreOpener-eb3adcb22c2a13d801d409bcb84d1d55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,285 INFO [StoreOpener-eb3adcb22c2a13d801d409bcb84d1d55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb3adcb22c2a13d801d409bcb84d1d55 columnFamilyName testFamily 2024-12-05T20:01:16,285 DEBUG [StoreOpener-eb3adcb22c2a13d801d409bcb84d1d55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:16,295 INFO [StoreOpener-eb3adcb22c2a13d801d409bcb84d1d55-1 {}] regionserver.HStore(327): Store=eb3adcb22c2a13d801d409bcb84d1d55/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:16,295 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,297 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,298 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,304 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,304 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,308 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:16,320 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened eb3adcb22c2a13d801d409bcb84d1d55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66189929, jitterRate=-0.013693198561668396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:16,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:01:16,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for eb3adcb22c2a13d801d409bcb84d1d55: Running coprocessor pre-open hook at 1733428876121Writing region info on filesystem at 1733428876121Initializing all the Stores at 1733428876123 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428876124 (+1 ms)Cleaning up temporary data from old regions at 1733428876304 (+180 ms)Running coprocessor post-open hooks at 1733428876320 (+16 ms)Region opened successfully at 1733428876322 (+2 ms) 2024-12-05T20:01:16,324 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., pid=9, masterSystemTime=1733428876024 2024-12-05T20:01:16,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:16,329 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:01:16,331 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eb3adcb22c2a13d801d409bcb84d1d55, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:16,355 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb3adcb22c2a13d801d409bcb84d1d55, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:16,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T20:01:16,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure eb3adcb22c2a13d801d409bcb84d1d55, server=86162e2766a8,39051,1733428836300 in 617 msec 2024-12-05T20:01:16,456 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:16,456 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428876456"}]},"ts":"1733428876456"} 2024-12-05T20:01:16,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T20:01:16,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=eb3adcb22c2a13d801d409bcb84d1d55, ASSIGN in 958 msec 2024-12-05T20:01:16,460 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-05T20:01:16,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:16,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.4650 sec 2024-12-05T20:01:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T20:01:18,033 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-05T20:01:18,033 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-05T20:01:18,034 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:18,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-05T20:01:18,048 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:18,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-05T20:01:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:18,074 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:18,074 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-05T20:01:18,092 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T20:01:18,095 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-05T20:01:21,077 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-05T20:01:21,619 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:24,109 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=18,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-12-05T20:01:24,110 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=18,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-12-05T20:01:24,110 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:01:24,166 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 219) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1155 (was 1165), ProcessCount=11 (was 11), AvailableMemoryMB=9725 (was 9821) 2024-12-05T20:01:24,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1155, ProcessCount=11, AvailableMemoryMB=9724 2024-12-05T20:01:24,200 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:24,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-05T20:01:24,205 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:24,206 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:24,207 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:24,209 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-05T20:01:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T20:01:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741839_1015 (size=412) 2024-12-05T20:01:24,509 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 122b10ec1c5bf38150f505d02b71c7d6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 18, skipping over this response of 89 bytes 2024-12-05T20:01:24,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741840_1016 (size=63) 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 122b10ec1c5bf38150f505d02b71c7d6, disabling compactions & flushes 2024-12-05T20:01:24,796 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. after waiting 0 ms 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:24,796 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:24,796 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 122b10ec1c5bf38150f505d02b71c7d6: Waiting for close lock at 1733428884796Disabling compacts and flushes for region at 1733428884796Disabling writes for close at 1733428884796Writing region close event to WAL at 1733428884796Closed at 1733428884796 2024-12-05T20:01:24,799 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:24,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733428884799"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428884799"}]},"ts":"1733428884799"} 2024-12-05T20:01:24,804 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:24,807 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:24,807 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428884807"}]},"ts":"1733428884807"} 2024-12-05T20:01:24,811 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-05T20:01:24,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=122b10ec1c5bf38150f505d02b71c7d6, ASSIGN}] 2024-12-05T20:01:24,814 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=122b10ec1c5bf38150f505d02b71c7d6, ASSIGN 2024-12-05T20:01:24,816 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=122b10ec1c5bf38150f505d02b71c7d6, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:24,974 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=122b10ec1c5bf38150f505d02b71c7d6, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:24,982 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=122b10ec1c5bf38150f505d02b71c7d6, ASSIGN because future has completed 2024-12-05T20:01:25,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 122b10ec1c5bf38150f505d02b71c7d6, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T20:01:25,399 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:25,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 122b10ec1c5bf38150f505d02b71c7d6, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:25,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:25,400 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,401 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,594 INFO [StoreOpener-122b10ec1c5bf38150f505d02b71c7d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,608 INFO [StoreOpener-122b10ec1c5bf38150f505d02b71c7d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 122b10ec1c5bf38150f505d02b71c7d6 columnFamilyName testFamily 2024-12-05T20:01:25,608 DEBUG [StoreOpener-122b10ec1c5bf38150f505d02b71c7d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:25,611 INFO [StoreOpener-122b10ec1c5bf38150f505d02b71c7d6-1 {}] regionserver.HStore(327): Store=122b10ec1c5bf38150f505d02b71c7d6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:25,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,619 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,660 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:25,680 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 122b10ec1c5bf38150f505d02b71c7d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66024925, jitterRate=-0.016151949763298035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:25,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:01:25,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 122b10ec1c5bf38150f505d02b71c7d6: Running coprocessor pre-open hook at 1733428885401Writing region info on filesystem at 1733428885401Initializing all the Stores at 1733428885409 (+8 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428885409Cleaning up temporary data from old regions at 1733428885622 (+213 ms)Running coprocessor post-open hooks at 1733428885680 (+58 ms)Region opened successfully at 1733428885681 (+1 ms) 2024-12-05T20:01:25,683 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., pid=12, masterSystemTime=1733428885263 2024-12-05T20:01:25,697 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=122b10ec1c5bf38150f505d02b71c7d6, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:25,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:25,698 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:01:25,713 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 122b10ec1c5bf38150f505d02b71c7d6, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:25,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T20:01:25,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 122b10ec1c5bf38150f505d02b71c7d6, server=86162e2766a8,39051,1733428836300 in 625 msec 2024-12-05T20:01:25,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T20:01:25,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=122b10ec1c5bf38150f505d02b71c7d6, ASSIGN in 948 msec 2024-12-05T20:01:25,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:25,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428885769"}]},"ts":"1733428885769"} 2024-12-05T20:01:25,773 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-05T20:01:25,778 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:25,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.5780 sec 2024-12-05T20:01:25,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-05T20:01:25,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:25,937 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-05T20:01:25,937 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:27,079 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-05T20:01:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-05T20:01:27,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-05T20:01:27,278 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-05T20:01:27,279 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:27,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-05T20:01:27,285 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:27,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-05T20:01:27,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:27,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:27,362 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-05T20:01:27,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T20:01:27,391 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-05T20:01:27,392 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T20:01:27,392 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T20:01:30,902 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:33,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007663' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T20:01:33,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55550 deadline: 1733428899393, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007663' does not exist 2024-12-05T20:01:33,398 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 86162e2766a8,39051,1733428836300 for closing scanner id = -7374406577480007663 for 122b10ec1c5bf38150f505d02b71c7d6 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007663' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T20:01:33,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1140 (was 1155), ProcessCount=11 (was 11), AvailableMemoryMB=9642 (was 9724) 2024-12-05T20:01:33,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1140, ProcessCount=11, AvailableMemoryMB=9641 2024-12-05T20:01:33,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-05T20:01:33,416 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:33,416 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:33,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-05T20:01:33,418 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T20:01:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741841_1017 (size=423) 2024-12-05T20:01:33,830 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2fbfa497a3b588507d6cf44d411c6415, NAME => 'testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741842_1018 (size=74) 2024-12-05T20:01:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 27, skipping over this response of 83 bytes 2024-12-05T20:01:34,240 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:34,240 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 2fbfa497a3b588507d6cf44d411c6415, disabling compactions & flushes 2024-12-05T20:01:34,240 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,240 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,240 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. after waiting 0 ms 2024-12-05T20:01:34,241 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,241 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,241 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2fbfa497a3b588507d6cf44d411c6415: Waiting for close lock at 1733428894240Disabling compacts and flushes for region at 1733428894240Disabling writes for close at 1733428894241 (+1 ms)Writing region close event to WAL at 1733428894241Closed at 1733428894241 2024-12-05T20:01:34,243 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:34,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733428894244"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428894244"}]},"ts":"1733428894244"} 2024-12-05T20:01:34,248 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:34,250 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:34,250 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428894250"}]},"ts":"1733428894250"} 2024-12-05T20:01:34,253 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-05T20:01:34,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2fbfa497a3b588507d6cf44d411c6415, ASSIGN}] 2024-12-05T20:01:34,256 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2fbfa497a3b588507d6cf44d411c6415, ASSIGN 2024-12-05T20:01:34,257 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2fbfa497a3b588507d6cf44d411c6415, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:34,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2fbfa497a3b588507d6cf44d411c6415, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:34,412 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2fbfa497a3b588507d6cf44d411c6415, ASSIGN because future has completed 2024-12-05T20:01:34,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbfa497a3b588507d6cf44d411c6415, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T20:01:34,436 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T20:01:34,572 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,572 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fbfa497a3b588507d6cf44d411c6415, NAME => 'testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:34,573 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,573 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:34,573 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,573 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,575 INFO [StoreOpener-2fbfa497a3b588507d6cf44d411c6415-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,578 INFO [StoreOpener-2fbfa497a3b588507d6cf44d411c6415-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbfa497a3b588507d6cf44d411c6415 columnFamilyName testFamily 2024-12-05T20:01:34,579 DEBUG [StoreOpener-2fbfa497a3b588507d6cf44d411c6415-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:34,580 INFO [StoreOpener-2fbfa497a3b588507d6cf44d411c6415-1 {}] regionserver.HStore(327): Store=2fbfa497a3b588507d6cf44d411c6415/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:34,580 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,581 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,582 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,582 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,582 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,585 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,589 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:34,590 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 2fbfa497a3b588507d6cf44d411c6415; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64449636, jitterRate=-0.039625585079193115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:34,590 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:01:34,591 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 2fbfa497a3b588507d6cf44d411c6415: Running coprocessor pre-open hook at 1733428894573Writing region info on filesystem at 1733428894573Initializing all the Stores at 1733428894575 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428894575Cleaning up temporary data from old regions at 1733428894582 (+7 ms)Running coprocessor post-open hooks at 1733428894590 (+8 ms)Region opened successfully at 1733428894591 (+1 ms) 2024-12-05T20:01:34,593 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415., pid=15, masterSystemTime=1733428894566 2024-12-05T20:01:34,603 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,604 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:01:34,605 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2fbfa497a3b588507d6cf44d411c6415, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:34,608 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbfa497a3b588507d6cf44d411c6415, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:34,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-05T20:01:34,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fbfa497a3b588507d6cf44d411c6415, server=86162e2766a8,39051,1733428836300 in 198 msec 2024-12-05T20:01:34,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T20:01:34,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2fbfa497a3b588507d6cf44d411c6415, ASSIGN in 360 msec 2024-12-05T20:01:34,619 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:34,619 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428894619"}]},"ts":"1733428894619"} 2024-12-05T20:01:34,622 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-05T20:01:34,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:34,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2110 sec 2024-12-05T20:01:35,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-05T20:01:35,934 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T20:01:36,442 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-05T20:01:36,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-05T20:01:36,443 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:36,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-05T20:01:36,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:36,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-05T20:01:36,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:36,463 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:36,463 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-05T20:01:36,466 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-05T20:01:36,468 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-05T20:01:36,469 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-05T20:01:36,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7374406577480007659 2024-12-05T20:01:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:55550 deadline: 1733428899468, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T20:01:36,473 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-05T20:01:36,474 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-05T20:01:36,475 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T20:01:36,476 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T20:01:36,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7374406577480007657 2024-12-05T20:01:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:55550 deadline: 1733428899475, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T20:01:36,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T20:01:36,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-7374406577480007656 2024-12-05T20:01:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:55550 deadline: 1733428899479, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T20:01:36,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T20:01:36,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-7374406577480007655 2024-12-05T20:01:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:55550 deadline: 1733428899483, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-05T20:01:36,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-05T20:01:36,499 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 223), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1057 (was 1140), ProcessCount=11 (was 11), AvailableMemoryMB=9608 (was 9641) 2024-12-05T20:01:36,508 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1057, ProcessCount=11, AvailableMemoryMB=9608 2024-12-05T20:01:36,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-05T20:01:36,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:36,514 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:36,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-05T20:01:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-05T20:01:36,515 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741843_1019 (size=418) 2024-12-05T20:01:36,928 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ba47246803f6d5c1850e9a0dfb2a9bc, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:36,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741844_1020 (size=69) 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 7ba47246803f6d5c1850e9a0dfb2a9bc, disabling compactions & flushes 2024-12-05T20:01:37,339 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. after waiting 0 ms 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,339 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,339 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ba47246803f6d5c1850e9a0dfb2a9bc: Waiting for close lock at 1733428897339Disabling compacts and flushes for region at 1733428897339Disabling writes for close at 1733428897339Writing region close event to WAL at 1733428897339Closed at 1733428897339 2024-12-05T20:01:37,342 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:37,342 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733428897342"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428897342"}]},"ts":"1733428897342"} 2024-12-05T20:01:37,347 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:37,349 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:37,349 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428897349"}]},"ts":"1733428897349"} 2024-12-05T20:01:37,352 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-05T20:01:37,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=7ba47246803f6d5c1850e9a0dfb2a9bc, ASSIGN}] 2024-12-05T20:01:37,354 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=7ba47246803f6d5c1850e9a0dfb2a9bc, ASSIGN 2024-12-05T20:01:37,355 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=7ba47246803f6d5c1850e9a0dfb2a9bc, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:37,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=7ba47246803f6d5c1850e9a0dfb2a9bc, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:37,511 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=7ba47246803f6d5c1850e9a0dfb2a9bc, ASSIGN because future has completed 2024-12-05T20:01:37,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba47246803f6d5c1850e9a0dfb2a9bc, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-05T20:01:37,670 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,671 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ba47246803f6d5c1850e9a0dfb2a9bc, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:37,671 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,671 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:37,671 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,671 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,678 INFO [StoreOpener-7ba47246803f6d5c1850e9a0dfb2a9bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,681 INFO [StoreOpener-7ba47246803f6d5c1850e9a0dfb2a9bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ba47246803f6d5c1850e9a0dfb2a9bc columnFamilyName testFamily 2024-12-05T20:01:37,681 DEBUG [StoreOpener-7ba47246803f6d5c1850e9a0dfb2a9bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:37,682 INFO [StoreOpener-7ba47246803f6d5c1850e9a0dfb2a9bc-1 {}] regionserver.HStore(327): Store=7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:37,682 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,683 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,684 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,685 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,685 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,687 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,690 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:37,691 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 7ba47246803f6d5c1850e9a0dfb2a9bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69409393, jitterRate=0.034280553460121155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:37,691 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:01:37,692 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 7ba47246803f6d5c1850e9a0dfb2a9bc: Running coprocessor pre-open hook at 1733428897672Writing region info on filesystem at 1733428897672Initializing all the Stores at 1733428897673 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428897673Cleaning up temporary data from old regions at 1733428897685 (+12 ms)Running coprocessor post-open hooks at 1733428897691 (+6 ms)Region opened successfully at 1733428897692 (+1 ms) 2024-12-05T20:01:37,693 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., pid=18, masterSystemTime=1733428897664 2024-12-05T20:01:37,697 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,697 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:01:37,698 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=7ba47246803f6d5c1850e9a0dfb2a9bc, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:37,701 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ba47246803f6d5c1850e9a0dfb2a9bc, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:37,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T20:01:37,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ba47246803f6d5c1850e9a0dfb2a9bc, server=86162e2766a8,39051,1733428836300 in 192 msec 2024-12-05T20:01:37,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-05T20:01:37,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=7ba47246803f6d5c1850e9a0dfb2a9bc, ASSIGN in 354 msec 2024-12-05T20:01:37,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:37,712 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428897712"}]},"ts":"1733428897712"} 2024-12-05T20:01:37,716 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-05T20:01:37,718 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:37,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2080 sec 2024-12-05T20:01:39,085 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-05T20:01:39,086 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-05T20:01:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-05T20:01:39,542 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-05T20:01:39,542 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-05T20:01:39,543 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:39,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-05T20:01:39,548 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:39,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-05T20:01:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:39,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:39,573 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-05T20:01:39,577 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T20:01:39,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T20:01:40,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:40,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=57,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-05T20:01:40,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=57,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-05T20:01:40,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:01:41,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 57, skipping over this response of 89 bytes 2024-12-05T20:01:41,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:41,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T20:01:42,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=59,methodName=Scan], waitTime=981ms, rpcTimeout=978ms 2024-12-05T20:01:42,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=59,methodName=Scan], waitTime=981ms, rpcTimeout=978ms 2024-12-05T20:01:42,580 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:01:42,595 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1012 (was 1057), ProcessCount=11 (was 11), AvailableMemoryMB=10816 (was 9608) - AvailableMemoryMB LEAK? - 2024-12-05T20:01:42,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1012, ProcessCount=11, AvailableMemoryMB=10815 2024-12-05T20:01:42,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-05T20:01:42,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:42,610 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:42,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-05T20:01:42,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T20:01:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741845_1021 (size=416) 2024-12-05T20:01:42,625 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7073f2049e40bfa8bcf40518ddaab62e, NAME => 'testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:42,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741846_1022 (size=67) 2024-12-05T20:01:43,036 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:43,037 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7073f2049e40bfa8bcf40518ddaab62e, disabling compactions & flushes 2024-12-05T20:01:43,037 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,037 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,037 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. after waiting 0 ms 2024-12-05T20:01:43,037 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,037 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,037 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7073f2049e40bfa8bcf40518ddaab62e: Waiting for close lock at 1733428903037Disabling compacts and flushes for region at 1733428903037Disabling writes for close at 1733428903037Writing region close event to WAL at 1733428903037Closed at 1733428903037 2024-12-05T20:01:43,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:43,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733428903041"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428903041"}]},"ts":"1733428903041"} 2024-12-05T20:01:43,045 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:43,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:43,048 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428903047"}]},"ts":"1733428903047"} 2024-12-05T20:01:43,051 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-05T20:01:43,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7073f2049e40bfa8bcf40518ddaab62e, ASSIGN}] 2024-12-05T20:01:43,054 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7073f2049e40bfa8bcf40518ddaab62e, ASSIGN 2024-12-05T20:01:43,056 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7073f2049e40bfa8bcf40518ddaab62e, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:43,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 59, skipping over this response of 89 bytes 2024-12-05T20:01:43,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7073f2049e40bfa8bcf40518ddaab62e, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:43,213 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7073f2049e40bfa8bcf40518ddaab62e, ASSIGN because future has completed 2024-12-05T20:01:43,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7073f2049e40bfa8bcf40518ddaab62e, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:43,385 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,386 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 7073f2049e40bfa8bcf40518ddaab62e, NAME => 'testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:43,387 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,387 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:43,387 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,387 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,390 INFO [StoreOpener-7073f2049e40bfa8bcf40518ddaab62e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,394 INFO [StoreOpener-7073f2049e40bfa8bcf40518ddaab62e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7073f2049e40bfa8bcf40518ddaab62e columnFamilyName testFamily 2024-12-05T20:01:43,394 DEBUG [StoreOpener-7073f2049e40bfa8bcf40518ddaab62e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:43,398 INFO [StoreOpener-7073f2049e40bfa8bcf40518ddaab62e-1 {}] regionserver.HStore(327): Store=7073f2049e40bfa8bcf40518ddaab62e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:43,399 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,400 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,401 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,402 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,402 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,405 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,424 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:43,425 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 7073f2049e40bfa8bcf40518ddaab62e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70297011, jitterRate=0.047507092356681824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:43,426 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:01:43,427 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 7073f2049e40bfa8bcf40518ddaab62e: Running coprocessor pre-open hook at 1733428903388Writing region info on filesystem at 1733428903388Initializing all the Stores at 1733428903390 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428903390Cleaning up temporary data from old regions at 1733428903402 (+12 ms)Running coprocessor post-open hooks at 1733428903426 (+24 ms)Region opened successfully at 1733428903426 2024-12-05T20:01:43,428 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., pid=21, masterSystemTime=1733428903378 2024-12-05T20:01:43,433 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,433 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:01:43,434 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7073f2049e40bfa8bcf40518ddaab62e, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:43,437 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7073f2049e40bfa8bcf40518ddaab62e, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:43,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-05T20:01:43,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7073f2049e40bfa8bcf40518ddaab62e, server=86162e2766a8,39051,1733428836300 in 225 msec 2024-12-05T20:01:43,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T20:01:43,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=7073f2049e40bfa8bcf40518ddaab62e, ASSIGN in 392 msec 2024-12-05T20:01:43,449 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:43,449 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428903449"}]},"ts":"1733428903449"} 2024-12-05T20:01:43,452 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-05T20:01:43,454 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:43,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 848 msec 2024-12-05T20:01:43,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T20:01:43,622 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-05T20:01:43,622 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-05T20:01:43,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:43,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-05T20:01:43,629 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:43,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-05T20:01:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:43,653 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:43,653 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-05T20:01:43,657 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T20:01:43,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T20:01:43,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T20:01:43,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-05T20:01:45,087 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-05T20:01:45,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-05T20:01:45,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:45,936 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-05T20:01:45,936 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:46,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007644' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T20:01:46,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55550 deadline: 1733428907673, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007644' does not exist 2024-12-05T20:01:46,674 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 86162e2766a8,39051,1733428836300 for closing scanner id = -7374406577480007644 for 7073f2049e40bfa8bcf40518ddaab62e of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007644' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T20:01:46,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-167491816_22 at /127.0.0.1:35662 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=971 (was 1012), ProcessCount=11 (was 11), AvailableMemoryMB=10555 (was 10815) 2024-12-05T20:01:46,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=971, ProcessCount=11, AvailableMemoryMB=10554 2024-12-05T20:01:46,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-05T20:01:46,697 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:46,697 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:46,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-05T20:01:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-05T20:01:46,699 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:46,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741847_1023 (size=411) 2024-12-05T20:01:47,115 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 148b55ed358af15b8953c807b829963a, NAME => 'testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741848_1024 (size=62) 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 148b55ed358af15b8953c807b829963a, disabling compactions & flushes 2024-12-05T20:01:47,125 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. after waiting 0 ms 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,125 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,125 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 148b55ed358af15b8953c807b829963a: Waiting for close lock at 1733428907125Disabling compacts and flushes for region at 1733428907125Disabling writes for close at 1733428907125Writing region close event to WAL at 1733428907125Closed at 1733428907125 2024-12-05T20:01:47,128 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:47,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733428907128"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428907128"}]},"ts":"1733428907128"} 2024-12-05T20:01:47,131 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:47,133 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:47,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428907133"}]},"ts":"1733428907133"} 2024-12-05T20:01:47,137 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-05T20:01:47,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=148b55ed358af15b8953c807b829963a, ASSIGN}] 2024-12-05T20:01:47,139 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=148b55ed358af15b8953c807b829963a, ASSIGN 2024-12-05T20:01:47,142 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=148b55ed358af15b8953c807b829963a, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 68, skipping over this response of 83 bytes 2024-12-05T20:01:47,293 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=148b55ed358af15b8953c807b829963a, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:47,297 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=148b55ed358af15b8953c807b829963a, ASSIGN because future has completed 2024-12-05T20:01:47,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 148b55ed358af15b8953c807b829963a, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:47,458 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,458 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 148b55ed358af15b8953c807b829963a, NAME => 'testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:47,459 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,459 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:47,459 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,459 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,461 INFO [StoreOpener-148b55ed358af15b8953c807b829963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,465 INFO [StoreOpener-148b55ed358af15b8953c807b829963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 148b55ed358af15b8953c807b829963a columnFamilyName testFamily 2024-12-05T20:01:47,465 DEBUG [StoreOpener-148b55ed358af15b8953c807b829963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:47,466 INFO [StoreOpener-148b55ed358af15b8953c807b829963a-1 {}] regionserver.HStore(327): Store=148b55ed358af15b8953c807b829963a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:47,466 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,468 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,469 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,470 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,470 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,473 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,481 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:47,482 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 148b55ed358af15b8953c807b829963a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71053230, jitterRate=0.058775633573532104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:47,482 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 148b55ed358af15b8953c807b829963a 2024-12-05T20:01:47,483 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 148b55ed358af15b8953c807b829963a: Running coprocessor pre-open hook at 1733428907459Writing region info on filesystem at 1733428907459Initializing all the Stores at 1733428907461 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428907461Cleaning up temporary data from old regions at 1733428907470 (+9 ms)Running coprocessor post-open hooks at 1733428907482 (+12 ms)Region opened successfully at 1733428907483 (+1 ms) 2024-12-05T20:01:47,485 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a., pid=24, masterSystemTime=1733428907451 2024-12-05T20:01:47,490 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=148b55ed358af15b8953c807b829963a, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:47,494 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,494 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:01:47,495 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 148b55ed358af15b8953c807b829963a, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:47,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T20:01:47,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 148b55ed358af15b8953c807b829963a, server=86162e2766a8,39051,1733428836300 in 200 msec 2024-12-05T20:01:47,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-05T20:01:47,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=148b55ed358af15b8953c807b829963a, ASSIGN in 364 msec 2024-12-05T20:01:47,506 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:47,507 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428907506"}]},"ts":"1733428907506"} 2024-12-05T20:01:47,510 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-05T20:01:47,512 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:47,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 819 msec 2024-12-05T20:01:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-05T20:01:47,712 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-05T20:01:47,713 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-05T20:01:47,713 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:47,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-05T20:01:47,725 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:47,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-05T20:01:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:47,748 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:47,748 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-05T20:01:47,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T20:01:47,754 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T20:01:47,756 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T20:01:47,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-05T20:01:48,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:50,515 INFO [master/86162e2766a8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T20:01:50,515 INFO [master/86162e2766a8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T20:01:50,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007640' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T20:01:50,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55550 deadline: 1733428911763, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007640' does not exist 2024-12-05T20:01:50,764 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 86162e2766a8,39051,1733428836300 for closing scanner id = -7374406577480007640 for 148b55ed358af15b8953c807b829963a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007640' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T20:01:50,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 223), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=909 (was 971), ProcessCount=11 (was 11), AvailableMemoryMB=10328 (was 10554) 2024-12-05T20:01:50,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=909, ProcessCount=11, AvailableMemoryMB=10327 2024-12-05T20:01:50,781 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:50,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-05T20:01:50,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:50,785 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:50,785 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-05T20:01:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T20:01:50,786 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:50,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741849_1025 (size=419) 2024-12-05T20:01:51,089 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-05T20:01:51,198 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d8672026f4959ddd90f06fc4dee7cec, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:01:51,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741850_1026 (size=70) 2024-12-05T20:01:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 8d8672026f4959ddd90f06fc4dee7cec, disabling compactions & flushes 2024-12-05T20:01:51,610 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. after waiting 0 ms 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,610 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,610 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d8672026f4959ddd90f06fc4dee7cec: Waiting for close lock at 1733428911610Disabling compacts and flushes for region at 1733428911610Disabling writes for close at 1733428911610Writing region close event to WAL at 1733428911610Closed at 1733428911610 2024-12-05T20:01:51,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:01:51,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733428911614"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428911614"}]},"ts":"1733428911614"} 2024-12-05T20:01:51,617 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:01:51,619 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:01:51,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428911619"}]},"ts":"1733428911619"} 2024-12-05T20:01:51,621 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-05T20:01:51,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8d8672026f4959ddd90f06fc4dee7cec, ASSIGN}] 2024-12-05T20:01:51,624 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8d8672026f4959ddd90f06fc4dee7cec, ASSIGN 2024-12-05T20:01:51,625 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8d8672026f4959ddd90f06fc4dee7cec, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:01:51,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8d8672026f4959ddd90f06fc4dee7cec, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:51,781 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8d8672026f4959ddd90f06fc4dee7cec, ASSIGN because future has completed 2024-12-05T20:01:51,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d8672026f4959ddd90f06fc4dee7cec, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:01:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T20:01:51,942 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,942 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d8672026f4959ddd90f06fc4dee7cec, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:01:51,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:01:51,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,948 INFO [StoreOpener-8d8672026f4959ddd90f06fc4dee7cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,951 INFO [StoreOpener-8d8672026f4959ddd90f06fc4dee7cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d8672026f4959ddd90f06fc4dee7cec columnFamilyName testFamily 2024-12-05T20:01:51,951 DEBUG [StoreOpener-8d8672026f4959ddd90f06fc4dee7cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:51,951 INFO [StoreOpener-8d8672026f4959ddd90f06fc4dee7cec-1 {}] regionserver.HStore(327): Store=8d8672026f4959ddd90f06fc4dee7cec/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:01:51,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,957 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,961 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:01:51,962 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 8d8672026f4959ddd90f06fc4dee7cec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58736742, jitterRate=-0.12475433945655823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:01:51,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:01:51,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 8d8672026f4959ddd90f06fc4dee7cec: Running coprocessor pre-open hook at 1733428911943Writing region info on filesystem at 1733428911943Initializing all the Stores at 1733428911945 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428911945Cleaning up temporary data from old regions at 1733428911954 (+9 ms)Running coprocessor post-open hooks at 1733428911963 (+9 ms)Region opened successfully at 1733428911963 2024-12-05T20:01:51,965 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., pid=27, masterSystemTime=1733428911936 2024-12-05T20:01:51,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,968 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:01:51,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8d8672026f4959ddd90f06fc4dee7cec, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:01:51,972 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d8672026f4959ddd90f06fc4dee7cec, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:01:51,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-05T20:01:51,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d8672026f4959ddd90f06fc4dee7cec, server=86162e2766a8,39051,1733428836300 in 191 msec 2024-12-05T20:01:51,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T20:01:51,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8d8672026f4959ddd90f06fc4dee7cec, ASSIGN in 355 msec 2024-12-05T20:01:51,982 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:01:51,982 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428911982"}]},"ts":"1733428911982"} 2024-12-05T20:01:51,985 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-05T20:01:51,986 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:01:51,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2050 sec 2024-12-05T20:01:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T20:01:53,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-05T20:01:53,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-05T20:01:53,833 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:53,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-05T20:01:53,838 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:01:53,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-05T20:01:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:01:53,852 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:01:53,852 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-05T20:01:53,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T20:01:53,857 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-05T20:01:55,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-05T20:01:55,934 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:55,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-05T20:01:55,935 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-05T20:01:57,445 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:01:57,487 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-05T20:01:59,860 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=86,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-12-05T20:01:59,860 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=86,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-12-05T20:01:59,860 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:01:59,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=844 (was 909), ProcessCount=11 (was 11), AvailableMemoryMB=10205 (was 10327) 2024-12-05T20:01:59,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=844, ProcessCount=11, AvailableMemoryMB=10205 2024-12-05T20:01:59,882 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:01:59,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-05T20:01:59,886 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:01:59,886 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:01:59,886 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-05T20:01:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T20:01:59,887 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:01:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741851_1027 (size=407) 2024-12-05T20:02:00,299 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e79255293d9ae9ce3ab135e8b9642b6a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:02:00,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741852_1028 (size=58) 2024-12-05T20:02:00,309 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:02:00,309 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e79255293d9ae9ce3ab135e8b9642b6a, disabling compactions & flushes 2024-12-05T20:02:00,309 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,309 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,309 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. after waiting 0 ms 2024-12-05T20:02:00,310 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,310 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,310 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e79255293d9ae9ce3ab135e8b9642b6a: Waiting for close lock at 1733428920309Disabling compacts and flushes for region at 1733428920309Disabling writes for close at 1733428920309Writing region close event to WAL at 1733428920310 (+1 ms)Closed at 1733428920310 2024-12-05T20:02:00,311 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:02:00,312 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733428920311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428920311"}]},"ts":"1733428920311"} 2024-12-05T20:02:00,315 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:02:00,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:02:00,316 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428920316"}]},"ts":"1733428920316"} 2024-12-05T20:02:00,319 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-05T20:02:00,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e79255293d9ae9ce3ab135e8b9642b6a, ASSIGN}] 2024-12-05T20:02:00,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e79255293d9ae9ce3ab135e8b9642b6a, ASSIGN 2024-12-05T20:02:00,322 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e79255293d9ae9ce3ab135e8b9642b6a, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:02:00,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 86, skipping over this response of 89 bytes 2024-12-05T20:02:00,473 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e79255293d9ae9ce3ab135e8b9642b6a, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:02:00,476 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e79255293d9ae9ce3ab135e8b9642b6a, ASSIGN because future has completed 2024-12-05T20:02:00,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e79255293d9ae9ce3ab135e8b9642b6a, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:02:00,635 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,635 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => e79255293d9ae9ce3ab135e8b9642b6a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:02:00,636 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,636 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:02:00,636 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,636 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,638 INFO [StoreOpener-e79255293d9ae9ce3ab135e8b9642b6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,640 INFO [StoreOpener-e79255293d9ae9ce3ab135e8b9642b6a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e79255293d9ae9ce3ab135e8b9642b6a columnFamilyName testFamily 2024-12-05T20:02:00,640 DEBUG [StoreOpener-e79255293d9ae9ce3ab135e8b9642b6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:02:00,641 INFO [StoreOpener-e79255293d9ae9ce3ab135e8b9642b6a-1 {}] regionserver.HStore(327): Store=e79255293d9ae9ce3ab135e8b9642b6a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:02:00,641 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,643 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,643 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,645 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,648 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:02:00,648 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened e79255293d9ae9ce3ab135e8b9642b6a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71878952, jitterRate=0.07107985019683838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:02:00,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:00,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for e79255293d9ae9ce3ab135e8b9642b6a: Running coprocessor pre-open hook at 1733428920636Writing region info on filesystem at 1733428920636Initializing all the Stores at 1733428920638 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428920638Cleaning up temporary data from old regions at 1733428920643 (+5 ms)Running coprocessor post-open hooks at 1733428920649 (+6 ms)Region opened successfully at 1733428920649 2024-12-05T20:02:00,650 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a., pid=30, masterSystemTime=1733428920629 2024-12-05T20:02:00,653 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,653 INFO [RS_OPEN_PRIORITY_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:00,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e79255293d9ae9ce3ab135e8b9642b6a, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:02:00,657 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e79255293d9ae9ce3ab135e8b9642b6a, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:02:00,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T20:02:00,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e79255293d9ae9ce3ab135e8b9642b6a, server=86162e2766a8,39051,1733428836300 in 182 msec 2024-12-05T20:02:00,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-05T20:02:00,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e79255293d9ae9ce3ab135e8b9642b6a, ASSIGN in 342 msec 2024-12-05T20:02:00,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:02:00,665 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428920665"}]},"ts":"1733428920665"} 2024-12-05T20:02:00,668 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-05T20:02:00,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:02:00,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 787 msec 2024-12-05T20:02:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T20:02:00,902 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-05T20:02:00,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-05T20:02:00,902 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:02:00,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-05T20:02:00,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:02:00,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-05T20:02:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:02:00,924 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:02:00,925 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-05T20:02:00,928 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-05T20:02:00,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-05T20:02:00,931 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-05T20:02:00,932 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-05T20:02:03,488 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-05T20:02:04,436 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T20:02:06,138 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:02:06,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-05T20:02:06,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-05T20:02:06,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007632' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-05T20:02:06,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39051 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:55550 deadline: 1733428932943, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007632' does not exist 2024-12-05T20:02:06,945 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 86162e2766a8,39051,1733428836300 for closing scanner id = -7374406577480007632 for e79255293d9ae9ce3ab135e8b9642b6a of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7374406577480007632' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T20:02:06,951 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=754 (was 844), ProcessCount=11 (was 11), AvailableMemoryMB=10076 (was 10205) 2024-12-05T20:02:06,960 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=754, ProcessCount=11, AvailableMemoryMB=10075 2024-12-05T20:02:06,962 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T20:02:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-05T20:02:06,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T20:02:06,966 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:02:06,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-05T20:02:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T20:02:06,967 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T20:02:06,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741853_1029 (size=423) 2024-12-05T20:02:07,377 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 55aea83d32a9fe9f6cd9bade88e12157, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc 2024-12-05T20:02:07,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741854_1030 (size=74) 2024-12-05T20:02:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 95, skipping over this response of 83 bytes 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 55aea83d32a9fe9f6cd9bade88e12157, disabling compactions & flushes 2024-12-05T20:02:07,786 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. after waiting 0 ms 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:07,786 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:07,786 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 55aea83d32a9fe9f6cd9bade88e12157: Waiting for close lock at 1733428927786Disabling compacts and flushes for region at 1733428927786Disabling writes for close at 1733428927786Writing region close event to WAL at 1733428927786Closed at 1733428927786 2024-12-05T20:02:07,788 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T20:02:07,788 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733428927788"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733428927788"}]},"ts":"1733428927788"} 2024-12-05T20:02:07,791 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T20:02:07,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T20:02:07,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428927792"}]},"ts":"1733428927792"} 2024-12-05T20:02:07,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-05T20:02:07,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=55aea83d32a9fe9f6cd9bade88e12157, ASSIGN}] 2024-12-05T20:02:07,796 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=55aea83d32a9fe9f6cd9bade88e12157, ASSIGN 2024-12-05T20:02:07,797 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=55aea83d32a9fe9f6cd9bade88e12157, ASSIGN; state=OFFLINE, location=86162e2766a8,39051,1733428836300; forceNewPlan=false, retain=false 2024-12-05T20:02:07,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=55aea83d32a9fe9f6cd9bade88e12157, regionState=OPENING, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:02:07,952 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=55aea83d32a9fe9f6cd9bade88e12157, ASSIGN because future has completed 2024-12-05T20:02:07,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55aea83d32a9fe9f6cd9bade88e12157, server=86162e2766a8,39051,1733428836300}] 2024-12-05T20:02:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T20:02:08,111 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:08,112 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 55aea83d32a9fe9f6cd9bade88e12157, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157.', STARTKEY => '', ENDKEY => ''} 2024-12-05T20:02:08,112 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,112 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T20:02:08,112 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,112 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,115 INFO [StoreOpener-55aea83d32a9fe9f6cd9bade88e12157-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,117 INFO [StoreOpener-55aea83d32a9fe9f6cd9bade88e12157-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55aea83d32a9fe9f6cd9bade88e12157 columnFamilyName testFamily 2024-12-05T20:02:08,117 DEBUG [StoreOpener-55aea83d32a9fe9f6cd9bade88e12157-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T20:02:08,117 INFO [StoreOpener-55aea83d32a9fe9f6cd9bade88e12157-1 {}] regionserver.HStore(327): Store=55aea83d32a9fe9f6cd9bade88e12157/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T20:02:08,117 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,118 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,119 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,119 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,120 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,122 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,125 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T20:02:08,126 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 55aea83d32a9fe9f6cd9bade88e12157; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69960246, jitterRate=0.04248890280723572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T20:02:08,126 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:08,126 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 55aea83d32a9fe9f6cd9bade88e12157: Running coprocessor pre-open hook at 1733428928113Writing region info on filesystem at 1733428928113Initializing all the Stores at 1733428928114 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733428928114Cleaning up temporary data from old regions at 1733428928120 (+6 ms)Running coprocessor post-open hooks at 1733428928126 (+6 ms)Region opened successfully at 1733428928126 2024-12-05T20:02:08,128 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., pid=33, masterSystemTime=1733428928105 2024-12-05T20:02:08,130 DEBUG [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:08,130 INFO [RS_OPEN_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:08,131 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=55aea83d32a9fe9f6cd9bade88e12157, regionState=OPEN, openSeqNum=2, regionLocation=86162e2766a8,39051,1733428836300 2024-12-05T20:02:08,134 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55aea83d32a9fe9f6cd9bade88e12157, server=86162e2766a8,39051,1733428836300 because future has completed 2024-12-05T20:02:08,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T20:02:08,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 55aea83d32a9fe9f6cd9bade88e12157, server=86162e2766a8,39051,1733428836300 in 183 msec 2024-12-05T20:02:08,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T20:02:08,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=55aea83d32a9fe9f6cd9bade88e12157, ASSIGN in 342 msec 2024-12-05T20:02:08,141 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T20:02:08,141 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733428928141"}]},"ts":"1733428928141"} 2024-12-05T20:02:08,144 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-05T20:02:08,145 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T20:02:08,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1830 sec 2024-12-05T20:02:09,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-05T20:02:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41067 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T20:02:09,992 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-05T20:02:09,992 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-05T20:02:09,992 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:02:09,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-05T20:02:09,997 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T20:02:09,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-05T20:02:10,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:02:10,011 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-05T20:02:10,011 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-05T20:02:10,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-05T20:02:10,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T20:02:10,874 DEBUG [master/86162e2766a8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T20:02:11,020 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=103,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-05T20:02:11,021 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=103,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-05T20:02:11,021 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:02:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 103, skipping over this response of 89 bytes 2024-12-05T20:02:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2] 2024-12-05T20:02:12,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39051 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-05T20:02:13,020 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=86162e2766a8:39051 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=105,methodName=Scan], waitTime=977ms, rpcTimeout=970ms 2024-12-05T20:02:13,021 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=105,methodName=Scan], waitTime=977ms, rpcTimeout=970ms 2024-12-05T20:02:13,021 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., hostname=86162e2766a8,39051,1733428836300, seqNum=2 from cache 2024-12-05T20:02:13,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=741 (was 754), ProcessCount=11 (was 11), AvailableMemoryMB=9977 (was 10075) 2024-12-05T20:02:13,031 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T20:02:13,031 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T20:02:13,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T20:02:13,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T20:02:13,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T20:02:13,037 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T20:02:13,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T20:02:13,037 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T20:02:13,037 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1030710501, stopped=false 2024-12-05T20:02:13,038 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=86162e2766a8,41067,1733428835413 2024-12-05T20:02:13,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T20:02:13,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T20:02:13,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:02:13,080 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T20:02:13,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:02:13,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T20:02:13,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T20:02:13,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T20:02:13,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T20:02:13,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,083 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '86162e2766a8,39051,1733428836300' ***** 2024-12-05T20:02:13,083 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T20:02:13,083 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T20:02:13,083 INFO [RS:0;86162e2766a8:39051 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T20:02:13,083 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T20:02:13,084 INFO [RS:0;86162e2766a8:39051 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T20:02:13,084 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 148b55ed358af15b8953c807b829963a 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 55aea83d32a9fe9f6cd9bade88e12157 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 9079d4b3f85f7c7f362b6f05e31b5422 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fbfa497a3b588507d6cf44d411c6415 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 7ba47246803f6d5c1850e9a0dfb2a9bc 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 7073f2049e40bfa8bcf40518ddaab62e 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 122b10ec1c5bf38150f505d02b71c7d6 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d8672026f4959ddd90f06fc4dee7cec 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(959): stopping server 86162e2766a8,39051,1733428836300 2024-12-05T20:02:13,085 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 148b55ed358af15b8953c807b829963a, disabling compactions & flushes 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T20:02:13,085 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:02:13,085 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:02:13,085 INFO [RS:0;86162e2766a8:39051 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;86162e2766a8:39051. 2024-12-05T20:02:13,085 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. after waiting 0 ms 2024-12-05T20:02:13,085 DEBUG [RS:0;86162e2766a8:39051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T20:02:13,085 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:02:13,086 DEBUG [RS:0;86162e2766a8:39051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:13,086 INFO [RS:0;86162e2766a8:39051 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T20:02:13,086 INFO [RS:0;86162e2766a8:39051 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T20:02:13,086 INFO [RS:0;86162e2766a8:39051 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T20:02:13,086 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T20:02:13,087 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-05T20:02:13,087 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T20:02:13,087 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T20:02:13,087 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T20:02:13,087 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T20:02:13,087 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T20:02:13,087 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 148b55ed358af15b8953c807b829963a, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a, eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:02:13,091 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-05T20:02:13,091 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 148b55ed358af15b8953c807b829963a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:13,154 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/.tmp/testFamily/b00e78a55e2948a8a08cb814b6323a51 is 61, key is row-0/testFamily:testQualifier/1733428907732/Put/seqid=0 2024-12-05T20:02:13,159 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/info/f1c1425fdfdf4e4cbaa3cef8824da733 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422./info:regioninfo/1733428873050/Put/seqid=0 2024-12-05T20:02:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741855_1031 (size=5049) 2024-12-05T20:02:13,167 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/.tmp/testFamily/b00e78a55e2948a8a08cb814b6323a51 2024-12-05T20:02:13,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741856_1032 (size=20564) 2024-12-05T20:02:13,212 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/.tmp/testFamily/b00e78a55e2948a8a08cb814b6323a51 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/testFamily/b00e78a55e2948a8a08cb814b6323a51 2024-12-05T20:02:13,222 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/testFamily/b00e78a55e2948a8a08cb814b6323a51, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:13,231 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 148b55ed358af15b8953c807b829963a in 141ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:13,237 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNext/148b55ed358af15b8953c807b829963a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:13,239 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:02:13,239 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 148b55ed358af15b8953c807b829963a: Waiting for close lock at 1733428933085Running coprocessor pre-close hooks at 1733428933085Disabling compacts and flushes for region at 1733428933085Disabling writes for close at 1733428933085Obtaining lock to block concurrent updates at 1733428933091 (+6 ms)Preparing flush snapshotting stores in 148b55ed358af15b8953c807b829963a at 1733428933091Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428933100 (+9 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. at 1733428933101 (+1 ms)Flushing 148b55ed358af15b8953c807b829963a/testFamily: creating writer at 1733428933103 (+2 ms)Flushing 148b55ed358af15b8953c807b829963a/testFamily: appending metadata at 1733428933149 (+46 ms)Flushing 148b55ed358af15b8953c807b829963a/testFamily: closing flushed file at 1733428933151 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa8957: reopening flushed file at 1733428933210 (+59 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 148b55ed358af15b8953c807b829963a in 141ms, sequenceid=8, compaction requested=false at 1733428933231 (+21 ms)Writing region close event to WAL at 1733428933232 (+1 ms)Running coprocessor post-close hooks at 1733428933238 (+6 ms)Closed at 1733428933239 (+1 ms) 2024-12-05T20:02:13,240 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733428906692.148b55ed358af15b8953c807b829963a. 2024-12-05T20:02:13,240 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb3adcb22c2a13d801d409bcb84d1d55, disabling compactions & flushes 2024-12-05T20:02:13,240 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:02:13,240 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:02:13,240 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. after waiting 0 ms 2024-12-05T20:02:13,240 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:02:13,240 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eb3adcb22c2a13d801d409bcb84d1d55 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:13,258 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/.tmp/testFamily/fc6f7cc10651447fa831397b69ec0c1c is 61, key is row-0/testFamily:testQualifier/1733428878056/Put/seqid=0 2024-12-05T20:02:13,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741857_1033 (size=5049) 2024-12-05T20:02:13,287 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a, eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:02:13,408 INFO [regionserver/86162e2766a8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T20:02:13,409 INFO [regionserver/86162e2766a8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T20:02:13,488 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a, eb3adcb22c2a13d801d409bcb84d1d55 2024-12-05T20:02:13,545 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-05T20:02:13,568 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/info/f1c1425fdfdf4e4cbaa3cef8824da733 2024-12-05T20:02:13,596 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/ns/b0cc434fb3214c9b8429c39f18066555 is 43, key is default/ns:d/1733428870105/Put/seqid=0 2024-12-05T20:02:13,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741858_1034 (size=5153) 2024-12-05T20:02:13,614 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:02:13,664 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/.tmp/testFamily/fc6f7cc10651447fa831397b69ec0c1c 2024-12-05T20:02:13,673 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/.tmp/testFamily/fc6f7cc10651447fa831397b69ec0c1c as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/testFamily/fc6f7cc10651447fa831397b69ec0c1c 2024-12-05T20:02:13,680 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/testFamily/fc6f7cc10651447fa831397b69ec0c1c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:13,681 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb3adcb22c2a13d801d409bcb84d1d55 in 441ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:13,686 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScanner/eb3adcb22c2a13d801d409bcb84d1d55/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:13,686 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:02:13,686 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb3adcb22c2a13d801d409bcb84d1d55: Waiting for close lock at 1733428933240Running coprocessor pre-close hooks at 1733428933240Disabling compacts and flushes for region at 1733428933240Disabling writes for close at 1733428933240Obtaining lock to block concurrent updates at 1733428933240Preparing flush snapshotting stores in eb3adcb22c2a13d801d409bcb84d1d55 at 1733428933240Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428933241 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. at 1733428933241Flushing eb3adcb22c2a13d801d409bcb84d1d55/testFamily: creating writer at 1733428933241Flushing eb3adcb22c2a13d801d409bcb84d1d55/testFamily: appending metadata at 1733428933258 (+17 ms)Flushing eb3adcb22c2a13d801d409bcb84d1d55/testFamily: closing flushed file at 1733428933258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f39aad4: reopening flushed file at 1733428933672 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eb3adcb22c2a13d801d409bcb84d1d55 in 441ms, sequenceid=8, compaction requested=false at 1733428933681 (+9 ms)Writing region close event to WAL at 1733428933682 (+1 ms)Running coprocessor post-close hooks at 1733428933686 (+4 ms)Closed at 1733428933686 2024-12-05T20:02:13,687 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733428874996.eb3adcb22c2a13d801d409bcb84d1d55. 2024-12-05T20:02:13,687 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55aea83d32a9fe9f6cd9bade88e12157, disabling compactions & flushes 2024-12-05T20:02:13,687 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:13,687 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:13,687 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. after waiting 0 ms 2024-12-05T20:02:13,687 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:13,687 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 55aea83d32a9fe9f6cd9bade88e12157 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:13,688 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:13,705 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/.tmp/testFamily/cacf5e950f6b4fd2b5350c2ca635b17e is 61, key is row-0/testFamily:testQualifier/1733428930001/Put/seqid=0 2024-12-05T20:02:13,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741859_1035 (size=5049) 2024-12-05T20:02:13,888 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:14,003 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/ns/b0cc434fb3214c9b8429c39f18066555 2024-12-05T20:02:14,039 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/table/5558225d4af341f880b12f69bf3957a7 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733428873233/Put/seqid=0 2024-12-05T20:02:14,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741860_1036 (size=6699) 2024-12-05T20:02:14,088 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-05T20:02:14,089 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1325): Online Regions={55aea83d32a9fe9f6cd9bade88e12157=testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., 9079d4b3f85f7c7f362b6f05e31b5422=testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422., 2fbfa497a3b588507d6cf44d411c6415=testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415., 1588230740=hbase:meta,,1.1588230740, 7ba47246803f6d5c1850e9a0dfb2a9bc=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., 7073f2049e40bfa8bcf40518ddaab62e=testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., 122b10ec1c5bf38150f505d02b71c7d6=hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., 8d8672026f4959ddd90f06fc4dee7cec=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., e79255293d9ae9ce3ab135e8b9642b6a=hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.} 2024-12-05T20:02:14,089 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 55aea83d32a9fe9f6cd9bade88e12157, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:14,111 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/.tmp/testFamily/cacf5e950f6b4fd2b5350c2ca635b17e 2024-12-05T20:02:14,121 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/.tmp/testFamily/cacf5e950f6b4fd2b5350c2ca635b17e as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/testFamily/cacf5e950f6b4fd2b5350c2ca635b17e 2024-12-05T20:02:14,130 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/testFamily/cacf5e950f6b4fd2b5350c2ca635b17e, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:14,131 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55aea83d32a9fe9f6cd9bade88e12157 in 444ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:14,138 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScannerAsync/55aea83d32a9fe9f6cd9bade88e12157/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:14,139 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55aea83d32a9fe9f6cd9bade88e12157: Waiting for close lock at 1733428933687Running coprocessor pre-close hooks at 1733428933687Disabling compacts and flushes for region at 1733428933687Disabling writes for close at 1733428933687Obtaining lock to block concurrent updates at 1733428933687Preparing flush snapshotting stores in 55aea83d32a9fe9f6cd9bade88e12157 at 1733428933687Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428933687Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. at 1733428933688 (+1 ms)Flushing 55aea83d32a9fe9f6cd9bade88e12157/testFamily: creating writer at 1733428933688Flushing 55aea83d32a9fe9f6cd9bade88e12157/testFamily: appending metadata at 1733428933704 (+16 ms)Flushing 55aea83d32a9fe9f6cd9bade88e12157/testFamily: closing flushed file at 1733428933704Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e34cb11: reopening flushed file at 1733428934120 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55aea83d32a9fe9f6cd9bade88e12157 in 444ms, sequenceid=8, compaction requested=false at 1733428934131 (+11 ms)Writing region close event to WAL at 1733428934133 (+2 ms)Running coprocessor post-close hooks at 1733428934138 (+5 ms)Closed at 1733428934139 (+1 ms) 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733428926962.55aea83d32a9fe9f6cd9bade88e12157. 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9079d4b3f85f7c7f362b6f05e31b5422, disabling compactions & flushes 2024-12-05T20:02:14,139 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. after waiting 0 ms 2024-12-05T20:02:14,139 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:02:14,139 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9079d4b3f85f7c7f362b6f05e31b5422 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:14,164 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/.tmp/testFamily/b902cd14845e470b80eae682519c6313 is 61, key is row-0/testFamily:testQualifier/1733428874436/Put/seqid=0 2024-12-05T20:02:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741861_1037 (size=5049) 2024-12-05T20:02:14,289 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:14,447 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/table/5558225d4af341f880b12f69bf3957a7 2024-12-05T20:02:14,457 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/info/f1c1425fdfdf4e4cbaa3cef8824da733 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/info/f1c1425fdfdf4e4cbaa3cef8824da733 2024-12-05T20:02:14,464 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/info/f1c1425fdfdf4e4cbaa3cef8824da733, entries=100, sequenceid=56, filesize=20.1 K 2024-12-05T20:02:14,465 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/ns/b0cc434fb3214c9b8429c39f18066555 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/ns/b0cc434fb3214c9b8429c39f18066555 2024-12-05T20:02:14,473 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/ns/b0cc434fb3214c9b8429c39f18066555, entries=2, sequenceid=56, filesize=5.0 K 2024-12-05T20:02:14,474 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/.tmp/table/5558225d4af341f880b12f69bf3957a7 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/table/5558225d4af341f880b12f69bf3957a7 2024-12-05T20:02:14,482 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/table/5558225d4af341f880b12f69bf3957a7, entries=20, sequenceid=56, filesize=6.5 K 2024-12-05T20:02:14,483 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1396ms, sequenceid=56, compaction requested=false 2024-12-05T20:02:14,489 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-05T20:02:14,489 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 1588230740, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:14,490 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T20:02:14,490 INFO [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T20:02:14,490 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733428933087Running coprocessor pre-close hooks at 1733428933087Disabling compacts and flushes for region at 1733428933087Disabling writes for close at 1733428933087Obtaining lock to block concurrent updates at 1733428933091 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1733428933091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733428933100 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733428933101 (+1 ms)Flushing 1588230740/info: creating writer at 1733428933103 (+2 ms)Flushing 1588230740/info: appending metadata at 1733428933155 (+52 ms)Flushing 1588230740/info: closing flushed file at 1733428933155Flushing 1588230740/ns: creating writer at 1733428933579 (+424 ms)Flushing 1588230740/ns: appending metadata at 1733428933595 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733428933595Flushing 1588230740/table: creating writer at 1733428934013 (+418 ms)Flushing 1588230740/table: appending metadata at 1733428934038 (+25 ms)Flushing 1588230740/table: closing flushed file at 1733428934038Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f5b14e6: reopening flushed file at 1733428934456 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cfe8b7: reopening flushed file at 1733428934464 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7045d5fe: reopening flushed file at 1733428934473 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1396ms, sequenceid=56, compaction requested=false at 1733428934483 (+10 ms)Writing region close event to WAL at 1733428934485 (+2 ms)Running coprocessor post-close hooks at 1733428934490 (+5 ms)Closed at 1733428934490 2024-12-05T20:02:14,490 DEBUG [RS_CLOSE_META-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T20:02:14,690 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, 9079d4b3f85f7c7f362b6f05e31b5422, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:14,813 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/.tmp/testFamily/b902cd14845e470b80eae682519c6313 2024-12-05T20:02:14,823 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/.tmp/testFamily/b902cd14845e470b80eae682519c6313 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/testFamily/b902cd14845e470b80eae682519c6313 2024-12-05T20:02:14,832 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/testFamily/b902cd14845e470b80eae682519c6313, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:14,833 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9079d4b3f85f7c7f362b6f05e31b5422 in 694ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:14,838 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextExceptionAsync/9079d4b3f85f7c7f362b6f05e31b5422/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:14,839 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:02:14,839 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9079d4b3f85f7c7f362b6f05e31b5422: Waiting for close lock at 1733428934139Running coprocessor pre-close hooks at 1733428934139Disabling compacts and flushes for region at 1733428934139Disabling writes for close at 1733428934139Obtaining lock to block concurrent updates at 1733428934139Preparing flush snapshotting stores in 9079d4b3f85f7c7f362b6f05e31b5422 at 1733428934139Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428934140 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. at 1733428934140Flushing 9079d4b3f85f7c7f362b6f05e31b5422/testFamily: creating writer at 1733428934140Flushing 9079d4b3f85f7c7f362b6f05e31b5422/testFamily: appending metadata at 1733428934164 (+24 ms)Flushing 9079d4b3f85f7c7f362b6f05e31b5422/testFamily: closing flushed file at 1733428934164Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c4ddd51: reopening flushed file at 1733428934822 (+658 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9079d4b3f85f7c7f362b6f05e31b5422 in 694ms, sequenceid=8, compaction requested=false at 1733428934834 (+12 ms)Writing region close event to WAL at 1733428934834Running coprocessor post-close hooks at 1733428934839 (+5 ms)Closed at 1733428934839 2024-12-05T20:02:14,839 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733428871113.9079d4b3f85f7c7f362b6f05e31b5422. 2024-12-05T20:02:14,840 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fbfa497a3b588507d6cf44d411c6415, disabling compactions & flushes 2024-12-05T20:02:14,840 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:02:14,840 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:02:14,840 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. after waiting 0 ms 2024-12-05T20:02:14,840 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:02:14,840 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2fbfa497a3b588507d6cf44d411c6415 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:14,861 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/.tmp/testFamily/02d96e22866247a991d1cf7131cf1d7f is 61, key is row-0/testFamily:testQualifier/1733428896453/Put/seqid=0 2024-12-05T20:02:14,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741862_1038 (size=5049) 2024-12-05T20:02:14,890 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,090 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-05T20:02:15,090 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1325): Online Regions={2fbfa497a3b588507d6cf44d411c6415=testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415., 7ba47246803f6d5c1850e9a0dfb2a9bc=testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., 7073f2049e40bfa8bcf40518ddaab62e=testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., 122b10ec1c5bf38150f505d02b71c7d6=hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., 8d8672026f4959ddd90f06fc4dee7cec=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., e79255293d9ae9ce3ab135e8b9642b6a=hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.} 2024-12-05T20:02:15,090 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,268 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/.tmp/testFamily/02d96e22866247a991d1cf7131cf1d7f 2024-12-05T20:02:15,278 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/.tmp/testFamily/02d96e22866247a991d1cf7131cf1d7f as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/testFamily/02d96e22866247a991d1cf7131cf1d7f 2024-12-05T20:02:15,285 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/testFamily/02d96e22866247a991d1cf7131cf1d7f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:15,286 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2fbfa497a3b588507d6cf44d411c6415 in 446ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:15,290 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testRetryOutOfOrderScannerNextException/2fbfa497a3b588507d6cf44d411c6415/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:15,291 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 2fbfa497a3b588507d6cf44d411c6415, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,291 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:02:15,291 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fbfa497a3b588507d6cf44d411c6415: Waiting for close lock at 1733428934840Running coprocessor pre-close hooks at 1733428934840Disabling compacts and flushes for region at 1733428934840Disabling writes for close at 1733428934840Obtaining lock to block concurrent updates at 1733428934840Preparing flush snapshotting stores in 2fbfa497a3b588507d6cf44d411c6415 at 1733428934840Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428934840Flushing stores of testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. at 1733428934841 (+1 ms)Flushing 2fbfa497a3b588507d6cf44d411c6415/testFamily: creating writer at 1733428934841Flushing 2fbfa497a3b588507d6cf44d411c6415/testFamily: appending metadata at 1733428934861 (+20 ms)Flushing 2fbfa497a3b588507d6cf44d411c6415/testFamily: closing flushed file at 1733428934861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54d12b0b: reopening flushed file at 1733428935277 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2fbfa497a3b588507d6cf44d411c6415 in 446ms, sequenceid=8, compaction requested=false at 1733428935286 (+9 ms)Writing region close event to WAL at 1733428935287 (+1 ms)Running coprocessor post-close hooks at 1733428935291 (+4 ms)Closed at 1733428935291 2024-12-05T20:02:15,291 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733428893411.2fbfa497a3b588507d6cf44d411c6415. 2024-12-05T20:02:15,292 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7ba47246803f6d5c1850e9a0dfb2a9bc, disabling compactions & flushes 2024-12-05T20:02:15,292 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:02:15,292 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:02:15,292 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. after waiting 0 ms 2024-12-05T20:02:15,292 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:02:15,292 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7ba47246803f6d5c1850e9a0dfb2a9bc 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:15,308 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/.tmp/testFamily/91144789e3284b9c85695f4d7f8ac3e3 is 61, key is row-0/testFamily:testQualifier/1733428899554/Put/seqid=0 2024-12-05T20:02:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741863_1039 (size=5049) 2024-12-05T20:02:15,491 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,691 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 7073f2049e40bfa8bcf40518ddaab62e, 7ba47246803f6d5c1850e9a0dfb2a9bc, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,795 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/.tmp/testFamily/91144789e3284b9c85695f4d7f8ac3e3 2024-12-05T20:02:15,804 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/.tmp/testFamily/91144789e3284b9c85695f4d7f8ac3e3 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily/91144789e3284b9c85695f4d7f8ac3e3 2024-12-05T20:02:15,812 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily/91144789e3284b9c85695f4d7f8ac3e3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:15,813 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ba47246803f6d5c1850e9a0dfb2a9bc in 521ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:15,819 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnOpenScanner/7ba47246803f6d5c1850e9a0dfb2a9bc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:15,819 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7ba47246803f6d5c1850e9a0dfb2a9bc: Waiting for close lock at 1733428935291Running coprocessor pre-close hooks at 1733428935291Disabling compacts and flushes for region at 1733428935292 (+1 ms)Disabling writes for close at 1733428935292Obtaining lock to block concurrent updates at 1733428935292Preparing flush snapshotting stores in 7ba47246803f6d5c1850e9a0dfb2a9bc at 1733428935292Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428935292Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. at 1733428935293 (+1 ms)Flushing 7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily: creating writer at 1733428935293Flushing 7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily: appending metadata at 1733428935307 (+14 ms)Flushing 7ba47246803f6d5c1850e9a0dfb2a9bc/testFamily: closing flushed file at 1733428935307Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45e83e3a: reopening flushed file at 1733428935803 (+496 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7ba47246803f6d5c1850e9a0dfb2a9bc in 521ms, sequenceid=8, compaction requested=false at 1733428935814 (+11 ms)Writing region close event to WAL at 1733428935814Running coprocessor post-close hooks at 1733428935819 (+5 ms)Closed at 1733428935819 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733428896510.7ba47246803f6d5c1850e9a0dfb2a9bc. 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7073f2049e40bfa8bcf40518ddaab62e, disabling compactions & flushes 2024-12-05T20:02:15,820 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. after waiting 0 ms 2024-12-05T20:02:15,820 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:02:15,820 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7073f2049e40bfa8bcf40518ddaab62e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:15,838 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/.tmp/testFamily/09aa8dd5d730477c8e029fddc420413c is 61, key is row-0/testFamily:testQualifier/1733428903634/Put/seqid=0 2024-12-05T20:02:15,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741864_1040 (size=5049) 2024-12-05T20:02:15,892 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 7073f2049e40bfa8bcf40518ddaab62e, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:15,934 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-05T20:02:15,935 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-05T20:02:16,092 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-05T20:02:16,092 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1325): Online Regions={7073f2049e40bfa8bcf40518ddaab62e=testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., 122b10ec1c5bf38150f505d02b71c7d6=hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., 8d8672026f4959ddd90f06fc4dee7cec=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., e79255293d9ae9ce3ab135e8b9642b6a=hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.} 2024-12-05T20:02:16,092 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 7073f2049e40bfa8bcf40518ddaab62e, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:16,249 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/.tmp/testFamily/09aa8dd5d730477c8e029fddc420413c 2024-12-05T20:02:16,258 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/.tmp/testFamily/09aa8dd5d730477c8e029fddc420413c as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/testFamily/09aa8dd5d730477c8e029fddc420413c 2024-12-05T20:02:16,265 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/testFamily/09aa8dd5d730477c8e029fddc420413c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:16,266 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7073f2049e40bfa8bcf40518ddaab62e in 446ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:16,270 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=86162e2766a8:41067 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-12-05T20:02:16,273 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/default/testNormalScanTimeoutOnNextAsync/7073f2049e40bfa8bcf40518ddaab62e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:16,273 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:02:16,273 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7073f2049e40bfa8bcf40518ddaab62e: Waiting for close lock at 1733428935820Running coprocessor pre-close hooks at 1733428935820Disabling compacts and flushes for region at 1733428935820Disabling writes for close at 1733428935820Obtaining lock to block concurrent updates at 1733428935820Preparing flush snapshotting stores in 7073f2049e40bfa8bcf40518ddaab62e at 1733428935820Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428935820Flushing stores of testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. at 1733428935821 (+1 ms)Flushing 7073f2049e40bfa8bcf40518ddaab62e/testFamily: creating writer at 1733428935821Flushing 7073f2049e40bfa8bcf40518ddaab62e/testFamily: appending metadata at 1733428935838 (+17 ms)Flushing 7073f2049e40bfa8bcf40518ddaab62e/testFamily: closing flushed file at 1733428935838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444db70d: reopening flushed file at 1733428936257 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7073f2049e40bfa8bcf40518ddaab62e in 446ms, sequenceid=8, compaction requested=false at 1733428936266 (+9 ms)Writing region close event to WAL at 1733428936267 (+1 ms)Running coprocessor post-close hooks at 1733428936273 (+6 ms)Closed at 1733428936273 2024-12-05T20:02:16,274 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733428902606.7073f2049e40bfa8bcf40518ddaab62e. 2024-12-05T20:02:16,274 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 122b10ec1c5bf38150f505d02b71c7d6, disabling compactions & flushes 2024-12-05T20:02:16,274 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:02:16,274 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:02:16,274 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. after waiting 0 ms 2024-12-05T20:02:16,274 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:02:16,274 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 122b10ec1c5bf38150f505d02b71c7d6 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:16,292 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/.tmp/testFamily/e8101b3bfc044cea808985e4f13a7b88 is 61, key is row-0/testFamily:testQualifier/1733428887292/Put/seqid=0 2024-12-05T20:02:16,292 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 122b10ec1c5bf38150f505d02b71c7d6, 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741865_1041 (size=5049) 2024-12-05T20:02:16,300 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/.tmp/testFamily/e8101b3bfc044cea808985e4f13a7b88 2024-12-05T20:02:16,309 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/.tmp/testFamily/e8101b3bfc044cea808985e4f13a7b88 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/testFamily/e8101b3bfc044cea808985e4f13a7b88 2024-12-05T20:02:16,316 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/testFamily/e8101b3bfc044cea808985e4f13a7b88, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:16,317 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 122b10ec1c5bf38150f505d02b71c7d6 in 43ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:16,322 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNextAsync/122b10ec1c5bf38150f505d02b71c7d6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:16,323 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:02:16,323 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 122b10ec1c5bf38150f505d02b71c7d6: Waiting for close lock at 1733428936274Running coprocessor pre-close hooks at 1733428936274Disabling compacts and flushes for region at 1733428936274Disabling writes for close at 1733428936274Obtaining lock to block concurrent updates at 1733428936274Preparing flush snapshotting stores in 122b10ec1c5bf38150f505d02b71c7d6 at 1733428936274Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428936274Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. at 1733428936275 (+1 ms)Flushing 122b10ec1c5bf38150f505d02b71c7d6/testFamily: creating writer at 1733428936275Flushing 122b10ec1c5bf38150f505d02b71c7d6/testFamily: appending metadata at 1733428936292 (+17 ms)Flushing 122b10ec1c5bf38150f505d02b71c7d6/testFamily: closing flushed file at 1733428936292Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@caad733: reopening flushed file at 1733428936308 (+16 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 122b10ec1c5bf38150f505d02b71c7d6 in 43ms, sequenceid=8, compaction requested=false at 1733428936317 (+9 ms)Writing region close event to WAL at 1733428936318 (+1 ms)Running coprocessor post-close hooks at 1733428936323 (+5 ms)Closed at 1733428936323 2024-12-05T20:02:16,323 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733428884200.122b10ec1c5bf38150f505d02b71c7d6. 2024-12-05T20:02:16,324 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d8672026f4959ddd90f06fc4dee7cec, disabling compactions & flushes 2024-12-05T20:02:16,324 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:02:16,324 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:02:16,324 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. after waiting 0 ms 2024-12-05T20:02:16,324 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:02:16,324 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8d8672026f4959ddd90f06fc4dee7cec 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:16,348 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/.tmp/testFamily/369b332aa4024e2d9e04ebc1c55de847 is 61, key is row-0/testFamily:testQualifier/1733428913843/Put/seqid=0 2024-12-05T20:02:16,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741866_1042 (size=5049) 2024-12-05T20:02:16,493 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:16,693 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on 8d8672026f4959ddd90f06fc4dee7cec, e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:16,754 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/.tmp/testFamily/369b332aa4024e2d9e04ebc1c55de847 2024-12-05T20:02:16,763 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/.tmp/testFamily/369b332aa4024e2d9e04ebc1c55de847 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/testFamily/369b332aa4024e2d9e04ebc1c55de847 2024-12-05T20:02:16,771 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/testFamily/369b332aa4024e2d9e04ebc1c55de847, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:16,772 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8d8672026f4959ddd90f06fc4dee7cec in 448ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:16,778 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8d8672026f4959ddd90f06fc4dee7cec/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:16,779 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d8672026f4959ddd90f06fc4dee7cec: Waiting for close lock at 1733428936324Running coprocessor pre-close hooks at 1733428936324Disabling compacts and flushes for region at 1733428936324Disabling writes for close at 1733428936324Obtaining lock to block concurrent updates at 1733428936324Preparing flush snapshotting stores in 8d8672026f4959ddd90f06fc4dee7cec at 1733428936324Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428936324Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. at 1733428936325 (+1 ms)Flushing 8d8672026f4959ddd90f06fc4dee7cec/testFamily: creating writer at 1733428936325Flushing 8d8672026f4959ddd90f06fc4dee7cec/testFamily: appending metadata at 1733428936348 (+23 ms)Flushing 8d8672026f4959ddd90f06fc4dee7cec/testFamily: closing flushed file at 1733428936348Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c517aa: reopening flushed file at 1733428936762 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8d8672026f4959ddd90f06fc4dee7cec in 448ms, sequenceid=8, compaction requested=false at 1733428936772 (+10 ms)Writing region close event to WAL at 1733428936773 (+1 ms)Running coprocessor post-close hooks at 1733428936778 (+5 ms)Closed at 1733428936778 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733428910780.8d8672026f4959ddd90f06fc4dee7cec. 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e79255293d9ae9ce3ab135e8b9642b6a, disabling compactions & flushes 2024-12-05T20:02:16,779 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. after waiting 0 ms 2024-12-05T20:02:16,779 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:16,779 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e79255293d9ae9ce3ab135e8b9642b6a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-05T20:02:16,798 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/.tmp/testFamily/6e15539abfe84d0584bbc1be430ef8ce is 61, key is row-0/testFamily:testQualifier/1733428920914/Put/seqid=0 2024-12-05T20:02:16,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741867_1043 (size=5049) 2024-12-05T20:02:16,893 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:17,094 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T20:02:17,094 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1325): Online Regions={e79255293d9ae9ce3ab135e8b9642b6a=hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a.} 2024-12-05T20:02:17,094 DEBUG [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1351): Waiting on e79255293d9ae9ce3ab135e8b9642b6a 2024-12-05T20:02:17,207 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/.tmp/testFamily/6e15539abfe84d0584bbc1be430ef8ce 2024-12-05T20:02:17,216 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/.tmp/testFamily/6e15539abfe84d0584bbc1be430ef8ce as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/testFamily/6e15539abfe84d0584bbc1be430ef8ce 2024-12-05T20:02:17,224 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/testFamily/6e15539abfe84d0584bbc1be430ef8ce, entries=4, sequenceid=8, filesize=4.9 K 2024-12-05T20:02:17,225 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e79255293d9ae9ce3ab135e8b9642b6a in 446ms, sequenceid=8, compaction requested=false 2024-12-05T20:02:17,230 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/data/hbase/testMetaScanTimeoutOnNext/e79255293d9ae9ce3ab135e8b9642b6a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-05T20:02:17,231 INFO [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:17,231 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e79255293d9ae9ce3ab135e8b9642b6a: Waiting for close lock at 1733428936779Running coprocessor pre-close hooks at 1733428936779Disabling compacts and flushes for region at 1733428936779Disabling writes for close at 1733428936779Obtaining lock to block concurrent updates at 1733428936779Preparing flush snapshotting stores in e79255293d9ae9ce3ab135e8b9642b6a at 1733428936779Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733428936780 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. at 1733428936780Flushing e79255293d9ae9ce3ab135e8b9642b6a/testFamily: creating writer at 1733428936780Flushing e79255293d9ae9ce3ab135e8b9642b6a/testFamily: appending metadata at 1733428936797 (+17 ms)Flushing e79255293d9ae9ce3ab135e8b9642b6a/testFamily: closing flushed file at 1733428936797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31ed0b92: reopening flushed file at 1733428937215 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e79255293d9ae9ce3ab135e8b9642b6a in 446ms, sequenceid=8, compaction requested=false at 1733428937225 (+10 ms)Writing region close event to WAL at 1733428937226 (+1 ms)Running coprocessor post-close hooks at 1733428937231 (+5 ms)Closed at 1733428937231 2024-12-05T20:02:17,231 DEBUG [RS_CLOSE_REGION-regionserver/86162e2766a8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733428919882.e79255293d9ae9ce3ab135e8b9642b6a. 2024-12-05T20:02:17,294 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(976): stopping server 86162e2766a8,39051,1733428836300; all regions closed. 2024-12-05T20:02:17,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741834_1010 (size=19479) 2024-12-05T20:02:17,306 DEBUG [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/oldWALs 2024-12-05T20:02:17,306 INFO [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C39051%2C1733428836300.meta:.meta(num 1733428869215) 2024-12-05T20:02:17,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741833_1009 (size=20031) 2024-12-05T20:02:17,309 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/WALs/86162e2766a8,39051,1733428836300/86162e2766a8%2C39051%2C1733428836300.1733428868523 not finished, retry = 0 2024-12-05T20:02:17,413 DEBUG [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/oldWALs 2024-12-05T20:02:17,413 INFO [RS:0;86162e2766a8:39051 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 86162e2766a8%2C39051%2C1733428836300:(num 1733428868523) 2024-12-05T20:02:17,413 DEBUG [RS:0;86162e2766a8:39051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T20:02:17,413 INFO [RS:0;86162e2766a8:39051 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T20:02:17,413 INFO [RS:0;86162e2766a8:39051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T20:02:17,413 INFO [RS:0;86162e2766a8:39051 {}] hbase.ChoreService(370): Chore service for: regionserver/86162e2766a8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T20:02:17,414 INFO [RS:0;86162e2766a8:39051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T20:02:17,414 INFO [regionserver/86162e2766a8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T20:02:17,415 INFO [RS:0;86162e2766a8:39051 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39051 2024-12-05T20:02:17,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/86162e2766a8,39051,1733428836300 2024-12-05T20:02:17,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T20:02:17,420 INFO [RS:0;86162e2766a8:39051 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T20:02:17,422 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [86162e2766a8,39051,1733428836300] 2024-12-05T20:02:17,424 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/86162e2766a8,39051,1733428836300 already deleted, retry=false 2024-12-05T20:02:17,424 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 86162e2766a8,39051,1733428836300 expired; onlineServers=0 2024-12-05T20:02:17,424 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '86162e2766a8,41067,1733428835413' ***** 2024-12-05T20:02:17,424 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T20:02:17,424 INFO [M:0;86162e2766a8:41067 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T20:02:17,425 INFO [M:0;86162e2766a8:41067 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T20:02:17,425 DEBUG [M:0;86162e2766a8:41067 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T20:02:17,425 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T20:02:17,425 DEBUG [M:0;86162e2766a8:41067 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T20:02:17,425 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733428838286 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.large.0-1733428838286,5,FailOnTimeoutGroup] 2024-12-05T20:02:17,425 DEBUG [master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733428838291 {}] cleaner.HFileCleaner(306): Exit Thread[master/86162e2766a8:0:becomeActiveMaster-HFileCleaner.small.0-1733428838291,5,FailOnTimeoutGroup] 2024-12-05T20:02:17,425 INFO [M:0;86162e2766a8:41067 {}] hbase.ChoreService(370): Chore service for: master/86162e2766a8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T20:02:17,425 INFO [M:0;86162e2766a8:41067 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T20:02:17,425 DEBUG [M:0;86162e2766a8:41067 {}] master.HMaster(1795): Stopping service threads 2024-12-05T20:02:17,425 INFO [M:0;86162e2766a8:41067 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T20:02:17,426 INFO [M:0;86162e2766a8:41067 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T20:02:17,426 INFO [M:0;86162e2766a8:41067 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T20:02:17,426 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T20:02:17,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T20:02:17,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T20:02:17,427 DEBUG [M:0;86162e2766a8:41067 {}] zookeeper.ZKUtil(347): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T20:02:17,427 WARN [M:0;86162e2766a8:41067 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T20:02:17,428 INFO [M:0;86162e2766a8:41067 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/.lastflushedseqids 2024-12-05T20:02:17,430 INFO [regionserver/86162e2766a8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T20:02:17,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741868_1044 (size=721) 2024-12-05T20:02:17,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T20:02:17,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x10063c4a9b50001, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T20:02:17,523 INFO [RS:0;86162e2766a8:39051 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T20:02:17,523 INFO [RS:0;86162e2766a8:39051 {}] regionserver.HRegionServer(1031): Exiting; stopping=86162e2766a8,39051,1733428836300; zookeeper connection closed. 2024-12-05T20:02:17,524 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b634299 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b634299 2024-12-05T20:02:17,524 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T20:02:17,841 INFO [M:0;86162e2766a8:41067 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T20:02:17,841 INFO [M:0;86162e2766a8:41067 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T20:02:17,841 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T20:02:17,841 INFO [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:02:17,841 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:02:17,841 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T20:02:17,841 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:02:17,842 INFO [M:0;86162e2766a8:41067 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.15 KB heapSize=191.20 KB 2024-12-05T20:02:17,860 DEBUG [M:0;86162e2766a8:41067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d838ae467bd4920b4bd5d54408d9bb2 is 82, key is hbase:meta,,1/info:regioninfo/1733428869596/Put/seqid=0 2024-12-05T20:02:17,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741869_1045 (size=5672) 2024-12-05T20:02:18,266 INFO [M:0;86162e2766a8:41067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d838ae467bd4920b4bd5d54408d9bb2 2024-12-05T20:02:18,294 DEBUG [M:0;86162e2766a8:41067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41765d3544a5455b9dd50dc52c2641c0 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733428873283/Put/seqid=0 2024-12-05T20:02:18,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741870_1046 (size=14076) 2024-12-05T20:02:18,700 INFO [M:0;86162e2766a8:41067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.59 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41765d3544a5455b9dd50dc52c2641c0 2024-12-05T20:02:18,708 INFO [M:0;86162e2766a8:41067 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41765d3544a5455b9dd50dc52c2641c0 2024-12-05T20:02:18,728 DEBUG [M:0;86162e2766a8:41067 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86207848242540d2b71be10707dc7dd4 is 69, key is 86162e2766a8,39051,1733428836300/rs:state/1733428838324/Put/seqid=0 2024-12-05T20:02:18,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741871_1047 (size=5156) 2024-12-05T20:02:18,741 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T20:02:19,134 INFO [M:0;86162e2766a8:41067 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86207848242540d2b71be10707dc7dd4 2024-12-05T20:02:19,142 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3d838ae467bd4920b4bd5d54408d9bb2 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3d838ae467bd4920b4bd5d54408d9bb2 2024-12-05T20:02:19,149 INFO [M:0;86162e2766a8:41067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3d838ae467bd4920b4bd5d54408d9bb2, entries=8, sequenceid=319, filesize=5.5 K 2024-12-05T20:02:19,150 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41765d3544a5455b9dd50dc52c2641c0 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41765d3544a5455b9dd50dc52c2641c0 2024-12-05T20:02:19,156 INFO [M:0;86162e2766a8:41067 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41765d3544a5455b9dd50dc52c2641c0 2024-12-05T20:02:19,157 INFO [M:0;86162e2766a8:41067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41765d3544a5455b9dd50dc52c2641c0, entries=33, sequenceid=319, filesize=13.7 K 2024-12-05T20:02:19,158 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86207848242540d2b71be10707dc7dd4 as hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/86207848242540d2b71be10707dc7dd4 2024-12-05T20:02:19,164 INFO [M:0;86162e2766a8:41067 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41001/user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/86207848242540d2b71be10707dc7dd4, entries=1, sequenceid=319, filesize=5.0 K 2024-12-05T20:02:19,165 INFO [M:0;86162e2766a8:41067 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.15 KB/167063, heapSize ~190.90 KB/195480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1324ms, sequenceid=319, compaction requested=false 2024-12-05T20:02:19,166 INFO [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T20:02:19,166 DEBUG [M:0;86162e2766a8:41067 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733428937841Disabling compacts and flushes for region at 1733428937841Disabling writes for close at 1733428937841Obtaining lock to block concurrent updates at 1733428937842 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733428937842Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167063, getHeapSize=195720, getOffHeapSize=0, getCellsCount=366 at 1733428937842Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733428937843 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733428937843Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733428937860 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733428937860Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733428938274 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733428938293 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733428938293Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733428938708 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733428938727 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733428938727Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@670d2d12: reopening flushed file at 1733428939140 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a8171d: reopening flushed file at 1733428939149 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6384c3aa: reopening flushed file at 1733428939157 (+8 ms)Finished flush of dataSize ~163.15 KB/167063, heapSize ~190.90 KB/195480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1324ms, sequenceid=319, compaction requested=false at 1733428939165 (+8 ms)Writing region close event to WAL at 1733428939166 (+1 ms)Closed at 1733428939166 2024-12-05T20:02:19,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741830_1006 (size=188405) 2024-12-05T20:02:19,170 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a323b2d3-1f2d-2d36-106d-16dc303157cc/MasterData/WALs/86162e2766a8,41067,1733428835413/86162e2766a8%2C41067%2C1733428835413.1733428837640 not finished, retry = 0 2024-12-05T20:02:19,271 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T20:02:19,271 INFO [M:0;86162e2766a8:41067 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T20:02:19,272 INFO [M:0;86162e2766a8:41067 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41067 2024-12-05T20:02:19,272 INFO [M:0;86162e2766a8:41067 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T20:02:19,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T20:02:19,374 INFO [M:0;86162e2766a8:41067 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T20:02:19,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41067-0x10063c4a9b50000, quorum=127.0.0.1:51307, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T20:02:19,378 WARN [BP-1670072369-172.17.0.2-1733428832170 heartbeating to localhost/127.0.0.1:41001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1670072369-172.17.0.2-1733428832170 (Datanode Uuid 00a7e687-701c-469a-b475-6989e42e972e) service to localhost/127.0.0.1:41001 2024-12-05T20:02:19,379 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/data/data1/current/BP-1670072369-172.17.0.2-1733428832170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T20:02:19,380 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/cluster_b9394e63-6d69-3a4e-6f5e-9c4075ce745e/data/data2/current/BP-1670072369-172.17.0.2-1733428832170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T20:02:19,380 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T20:02:19,383 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T20:02:19,383 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T20:02:19,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T20:02:19,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.log.dir/,STOPPED} 2024-12-05T20:02:19,386 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T20:02:19,397 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T20:02:19,398 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T20:02:19,398 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T20:02:19,398 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T20:02:19,398 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ab595257-7c99-92c3-02ba-ed388ca1ce68/hadoop.log.dir/,STOPPED} 2024-12-05T20:02:19,409 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T20:02:19,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down