2024-11-20 17:58:57,628 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 17:58:57,652 main DEBUG Took 0.021940 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 17:58:57,653 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 17:58:57,653 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 17:58:57,654 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 17:58:57,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,664 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 17:58:57,680 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,683 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,684 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,685 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,687 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,688 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,689 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,690 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,690 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,692 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,693 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,694 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,696 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 17:58:57,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,697 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 17:58:57,699 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 17:58:57,700 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 17:58:57,703 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 17:58:57,704 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 17:58:57,706 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 17:58:57,706 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 17:58:57,717 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 17:58:57,720 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 17:58:57,722 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 17:58:57,723 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 17:58:57,723 main DEBUG createAppenders(={Console}) 2024-11-20 17:58:57,724 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-20 17:58:57,725 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-20 17:58:57,725 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-20 17:58:57,726 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 17:58:57,726 main DEBUG OutputStream closed 2024-11-20 17:58:57,726 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 17:58:57,727 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 17:58:57,727 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-20 17:58:57,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 17:58:57,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 17:58:57,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 17:58:57,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 17:58:57,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 17:58:57,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 17:58:57,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 17:58:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 17:58:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 17:58:57,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 17:58:57,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 17:58:57,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 17:58:57,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 17:58:57,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 17:58:57,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 17:58:57,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 17:58:57,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 17:58:57,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 17:58:57,833 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 17:58:57,833 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-20 17:58:57,834 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 17:58:57,835 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-20T17:58:57,848 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-20 17:58:57,851 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 17:58:57,851 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T17:58:58,112 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b 2024-11-20T17:58:58,134 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T17:58:58,155 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce, deleteOnExit=true 2024-11-20T17:58:58,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T17:58:58,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/test.cache.data in system properties and HBase conf 2024-11-20T17:58:58,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T17:58:58,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.log.dir in system properties and HBase conf 2024-11-20T17:58:58,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T17:58:58,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T17:58:58,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T17:58:58,258 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T17:58:58,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T17:58:58,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:58:58,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T17:58:58,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T17:58:58,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:58:58,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T17:58:58,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T17:58:58,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T17:58:58,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:58:58,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T17:58:58,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/nfs.dump.dir in system properties and HBase conf 2024-11-20T17:58:58,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/java.io.tmpdir in system properties and HBase conf 2024-11-20T17:58:58,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T17:58:58,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T17:58:58,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T17:58:59,545 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T17:58:59,652 INFO [Time-limited test {}] log.Log(170): Logging initialized @2795ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T17:58:59,742 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:58:59,811 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:58:59,836 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:58:59,836 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:58:59,838 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T17:58:59,852 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:58:59,855 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:58:59,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:59:00,067 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/java.io.tmpdir/jetty-localhost-40199-hadoop-hdfs-3_4_1-tests_jar-_-any-13378421503660664569/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T17:59:00,075 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:40199} 2024-11-20T17:59:00,076 INFO [Time-limited test {}] server.Server(415): Started @3220ms 2024-11-20T17:59:00,796 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T17:59:00,805 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T17:59:00,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T17:59:00,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T17:59:00,806 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T17:59:00,807 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6506e00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.log.dir/,AVAILABLE} 2024-11-20T17:59:00,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c25baf6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T17:59:00,920 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5359669a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/java.io.tmpdir/jetty-localhost-34965-hadoop-hdfs-3_4_1-tests_jar-_-any-16467688205168888620/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T17:59:00,921 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a8bacf4{HTTP/1.1, (http/1.1)}{localhost:34965} 2024-11-20T17:59:00,921 INFO [Time-limited test {}] server.Server(415): Started @4065ms 2024-11-20T17:59:00,977 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T17:59:02,070 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/data/data2/current/BP-1234759647-172.17.0.2-1732125539043/current, will proceed with Du for space computation calculation, 2024-11-20T17:59:02,070 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/data/data1/current/BP-1234759647-172.17.0.2-1732125539043/current, will proceed with Du for space computation calculation, 2024-11-20T17:59:02,102 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T17:59:02,157 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21886e36c893bba with lease ID 0x198fdc0210cb90d0: Processing first storage report for DS-aa4f7914-7084-4fbe-9372-838d0e479274 from datanode DatanodeRegistration(127.0.0.1:34453, datanodeUuid=22084a06-b039-4d4a-9b3b-52237c9bf74b, infoPort=46027, infoSecurePort=0, ipcPort=40587, storageInfo=lv=-57;cid=testClusterID;nsid=917447222;c=1732125539043) 2024-11-20T17:59:02,159 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21886e36c893bba with lease ID 0x198fdc0210cb90d0: from storage DS-aa4f7914-7084-4fbe-9372-838d0e479274 node DatanodeRegistration(127.0.0.1:34453, datanodeUuid=22084a06-b039-4d4a-9b3b-52237c9bf74b, infoPort=46027, infoSecurePort=0, ipcPort=40587, storageInfo=lv=-57;cid=testClusterID;nsid=917447222;c=1732125539043), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T17:59:02,159 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x21886e36c893bba with lease ID 0x198fdc0210cb90d0: Processing first storage report for DS-27e8723e-30dc-4d7d-a29c-e58c7a6f52a7 from datanode DatanodeRegistration(127.0.0.1:34453, datanodeUuid=22084a06-b039-4d4a-9b3b-52237c9bf74b, infoPort=46027, infoSecurePort=0, ipcPort=40587, storageInfo=lv=-57;cid=testClusterID;nsid=917447222;c=1732125539043) 2024-11-20T17:59:02,159 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x21886e36c893bba with lease ID 0x198fdc0210cb90d0: from storage DS-27e8723e-30dc-4d7d-a29c-e58c7a6f52a7 node DatanodeRegistration(127.0.0.1:34453, datanodeUuid=22084a06-b039-4d4a-9b3b-52237c9bf74b, infoPort=46027, infoSecurePort=0, ipcPort=40587, storageInfo=lv=-57;cid=testClusterID;nsid=917447222;c=1732125539043), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T17:59:02,168 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b 2024-11-20T17:59:02,278 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/zookeeper_0, clientPort=51016, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T17:59:02,287 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51016 2024-11-20T17:59:02,303 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:02,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741825_1001 (size=7) 2024-11-20T17:59:02,953 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 with version=8 2024-11-20T17:59:02,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/hbase-staging 2024-11-20T17:59:03,047 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T17:59:03,334 INFO [Time-limited test {}] client.ConnectionUtils(128): master/613581fcdc40:0 server-side Connection retries=9 2024-11-20T17:59:03,345 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:03,345 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:03,350 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:59:03,350 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:03,350 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:59:03,512 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T17:59:03,581 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T17:59:03,594 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T17:59:03,599 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:59:03,636 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 85201 (auto-detected) 2024-11-20T17:59:03,637 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T17:59:03,663 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35971 2024-11-20T17:59:03,687 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35971 connecting to ZooKeeper ensemble=127.0.0.1:51016 2024-11-20T17:59:03,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:359710x0, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:59:03,824 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35971-0x1015966431a0000 connected 2024-11-20T17:59:03,925 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:03,929 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:03,940 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:59:03,944 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04, hbase.cluster.distributed=false 2024-11-20T17:59:03,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:59:03,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-20T17:59:03,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35971 2024-11-20T17:59:03,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35971 2024-11-20T17:59:03,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-20T17:59:03,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-20T17:59:04,094 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/613581fcdc40:0 server-side Connection retries=9 2024-11-20T17:59:04,095 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:04,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:04,096 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T17:59:04,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T17:59:04,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T17:59:04,099 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T17:59:04,102 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T17:59:04,103 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46615 2024-11-20T17:59:04,105 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46615 connecting to ZooKeeper ensemble=127.0.0.1:51016 2024-11-20T17:59:04,106 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:04,110 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:04,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466150x0, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T17:59:04,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466150x0, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T17:59:04,131 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46615-0x1015966431a0001 connected 2024-11-20T17:59:04,135 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T17:59:04,144 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T17:59:04,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T17:59:04,152 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T17:59:04,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46615 2024-11-20T17:59:04,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46615 2024-11-20T17:59:04,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46615 2024-11-20T17:59:04,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46615 2024-11-20T17:59:04,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46615 2024-11-20T17:59:04,177 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;613581fcdc40:35971 2024-11-20T17:59:04,178 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/613581fcdc40,35971,1732125543153 2024-11-20T17:59:04,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:59:04,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:59:04,199 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/613581fcdc40,35971,1732125543153 2024-11-20T17:59:04,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T17:59:04,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:04,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:04,235 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T17:59:04,237 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/613581fcdc40,35971,1732125543153 from backup master directory 2024-11-20T17:59:04,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/613581fcdc40,35971,1732125543153 2024-11-20T17:59:04,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:59:04,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T17:59:04,250 WARN [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:59:04,250 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=613581fcdc40,35971,1732125543153 2024-11-20T17:59:04,253 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T17:59:04,255 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T17:59:04,312 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/hbase.id] with ID: 2e4bcc2a-a205-44fb-ba82-aeb59a139b49 2024-11-20T17:59:04,312 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/.tmp/hbase.id 2024-11-20T17:59:04,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741826_1002 (size=42) 2024-11-20T17:59:04,724 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/.tmp/hbase.id]:[hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/hbase.id] 2024-11-20T17:59:04,777 INFO [master/613581fcdc40:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T17:59:04,783 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T17:59:04,805 INFO [master/613581fcdc40:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-20T17:59:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:04,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741827_1003 (size=196) 2024-11-20T17:59:05,296 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T17:59:05,299 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T17:59:05,320 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T17:59:05,325 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:59:05,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741828_1004 (size=1189) 2024-11-20T17:59:05,780 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store 2024-11-20T17:59:05,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741829_1005 (size=34) 2024-11-20T17:59:06,209 INFO [master/613581fcdc40:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T17:59:06,213 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:59:06,215 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T17:59:06,215 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:59:06,215 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:59:06,217 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T17:59:06,217 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:59:06,217 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T17:59:06,218 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732125546215Disabling compacts and flushes for region at 1732125546215Disabling writes for close at 1732125546217 (+2 ms)Writing region close event to WAL at 1732125546217Closed at 1732125546217 2024-11-20T17:59:06,221 WARN [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/.initializing 2024-11-20T17:59:06,221 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/WALs/613581fcdc40,35971,1732125543153 2024-11-20T17:59:06,232 INFO [master/613581fcdc40:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T17:59:06,251 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C35971%2C1732125543153, suffix=, logDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/WALs/613581fcdc40,35971,1732125543153, archiveDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/oldWALs, maxLogs=10 2024-11-20T17:59:06,277 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/WALs/613581fcdc40,35971,1732125543153/613581fcdc40%2C35971%2C1732125543153.1732125546257, exclude list is [], retry=0 2024-11-20T17:59:06,295 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34453,DS-aa4f7914-7084-4fbe-9372-838d0e479274,DISK] 2024-11-20T17:59:06,299 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T17:59:06,338 INFO [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/WALs/613581fcdc40,35971,1732125543153/613581fcdc40%2C35971%2C1732125543153.1732125546257 2024-11-20T17:59:06,339 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46027:46027)] 2024-11-20T17:59:06,339 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T17:59:06,340 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:59:06,343 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,344 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,408 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T17:59:06,412 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:06,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:59:06,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T17:59:06,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:06,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:59:06,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T17:59:06,425 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:06,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:59:06,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T17:59:06,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:06,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T17:59:06,432 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,435 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,437 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,443 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,444 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,449 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:59:06,454 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T17:59:06,459 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:59:06,460 INFO [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71757117, jitterRate=0.06926436722278595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:59:06,469 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732125546356Initializing all the Stores at 1732125546359 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125546359Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125546360 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125546360Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125546360Cleaning up temporary data from old regions at 1732125546444 (+84 ms)Region opened successfully at 1732125546469 (+25 ms) 2024-11-20T17:59:06,471 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T17:59:06,509 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20eeb2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:59:06,546 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T17:59:06,558 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T17:59:06,558 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T17:59:06,561 INFO [master/613581fcdc40:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T17:59:06,562 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T17:59:06,569 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T17:59:06,569 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T17:59:06,604 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T17:59:06,617 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T17:59:06,701 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T17:59:06,704 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T17:59:06,706 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T17:59:06,718 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T17:59:06,721 INFO [master/613581fcdc40:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T17:59:06,726 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T17:59:06,733 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T17:59:06,735 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T17:59:06,743 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T17:59:06,763 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T17:59:06,775 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T17:59:06,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:59:06,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T17:59:06,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,789 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=613581fcdc40,35971,1732125543153, sessionid=0x1015966431a0000, setting cluster-up flag (Was=false) 2024-11-20T17:59:06,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,849 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T17:59:06,851 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,35971,1732125543153 2024-11-20T17:59:06,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T17:59:06,908 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T17:59:06,910 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=613581fcdc40,35971,1732125543153 2024-11-20T17:59:06,919 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T17:59:06,964 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(746): ClusterId : 2e4bcc2a-a205-44fb-ba82-aeb59a139b49 2024-11-20T17:59:06,967 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T17:59:06,984 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T17:59:06,984 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T17:59:06,998 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T17:59:06,999 DEBUG [RS:0;613581fcdc40:46615 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db56641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=613581fcdc40/172.17.0.2:0 2024-11-20T17:59:07,009 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T17:59:07,018 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;613581fcdc40:46615 2024-11-20T17:59:07,020 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T17:59:07,022 INFO [RS:0;613581fcdc40:46615 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T17:59:07,023 INFO [RS:0;613581fcdc40:46615 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T17:59:07,023 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T17:59:07,026 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,35971,1732125543153 with port=46615, startcode=1732125544057 2024-11-20T17:59:07,030 INFO [master/613581fcdc40:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T17:59:07,040 DEBUG [RS:0;613581fcdc40:46615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T17:59:07,037 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 613581fcdc40,35971,1732125543153 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T17:59:07,046 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:59:07,046 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:59:07,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:59:07,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/613581fcdc40:0, corePoolSize=5, maxPoolSize=5 2024-11-20T17:59:07,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/613581fcdc40:0, corePoolSize=10, maxPoolSize=10 2024-11-20T17:59:07,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:07,047 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:59:07,048 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:07,052 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732125577051 2024-11-20T17:59:07,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:59:07,054 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T17:59:07,054 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T17:59:07,055 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T17:59:07,059 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T17:59:07,060 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T17:59:07,060 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T17:59:07,060 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T17:59:07,061 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:07,062 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T17:59:07,072 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:07,081 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T17:59:07,082 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T17:59:07,082 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46631, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T17:59:07,083 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T17:59:07,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741831_1007 (size=1321) 2024-11-20T17:59:07,088 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T17:59:07,089 INFO [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T17:59:07,090 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35971 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T17:59:07,091 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T17:59:07,091 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T17:59:07,100 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732125547090,5,FailOnTimeoutGroup] 2024-11-20T17:59:07,101 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732125547101,5,FailOnTimeoutGroup] 2024-11-20T17:59:07,101 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:07,102 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T17:59:07,103 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:07,104 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:07,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741832_1008 (size=32) 2024-11-20T17:59:07,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T17:59:07,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T17:59:07,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T17:59:07,120 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T17:59:07,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:07,121 WARN [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-20T17:59:07,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:59:07,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T17:59:07,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T17:59:07,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:07,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:59:07,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T17:59:07,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T17:59:07,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:07,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:59:07,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T17:59:07,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T17:59:07,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T17:59:07,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T17:59:07,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T17:59:07,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740 2024-11-20T17:59:07,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740 2024-11-20T17:59:07,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T17:59:07,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T17:59:07,144 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T17:59:07,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T17:59:07,152 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T17:59:07,153 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65815676, jitterRate=-0.019270002841949463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T17:59:07,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732125547111Initializing all the Stores at 1732125547113 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125547114 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125547116 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125547116Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125547116Cleaning up temporary data from old regions at 1732125547143 (+27 ms)Region opened successfully at 1732125547157 (+14 ms) 2024-11-20T17:59:07,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T17:59:07,158 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T17:59:07,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T17:59:07,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T17:59:07,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T17:59:07,159 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T17:59:07,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732125547157Disabling compacts and flushes for region at 1732125547157Disabling writes for close at 1732125547158 (+1 ms)Writing region close event to WAL at 1732125547159 (+1 ms)Closed at 1732125547159 2024-11-20T17:59:07,163 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T17:59:07,164 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T17:59:07,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T17:59:07,178 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T17:59:07,182 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T17:59:07,333 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:08,334 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:09,335 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:10,335 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:11,336 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:12,337 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:13,338 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:13,577 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T17:59:13,578 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T17:59:14,339 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:15,339 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:16,340 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:17,341 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:18,342 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:19,342 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:20,343 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:21,344 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:22,345 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:23,345 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:24,346 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:25,347 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:26,348 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:27,348 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:28,349 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:29,350 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:30,350 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:31,351 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:32,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T17:59:32,352 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:33,353 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:34,354 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:35,354 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:36,355 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:37,122 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(2659): reportForDuty to master=613581fcdc40,35971,1732125543153 with port=46615, startcode=1732125544057 2024-11-20T17:59:37,126 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35971 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35971 {}] master.ServerManager(517): Registering regionserver=613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,139 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T17:59:37,140 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37129 2024-11-20T17:59:37,140 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T17:59:37,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T17:59:37,206 DEBUG [RS:0;613581fcdc40:46615 {}] zookeeper.ZKUtil(111): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,206 WARN [RS:0;613581fcdc40:46615 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T17:59:37,206 INFO [RS:0;613581fcdc40:46615 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T17:59:37,207 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,210 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [613581fcdc40,46615,1732125544057] 2024-11-20T17:59:37,233 INFO [RS:0;613581fcdc40:46615 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T17:59:37,247 INFO [RS:0;613581fcdc40:46615 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T17:59:37,253 INFO [RS:0;613581fcdc40:46615 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T17:59:37,253 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,254 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T17:59:37,260 INFO [RS:0;613581fcdc40:46615 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T17:59:37,262 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,263 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,263 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,263 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,263 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/613581fcdc40:0, corePoolSize=2, maxPoolSize=2 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,264 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,265 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,265 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/613581fcdc40:0, corePoolSize=1, maxPoolSize=1 2024-11-20T17:59:37,265 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:59:37,265 DEBUG [RS:0;613581fcdc40:46615 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/613581fcdc40:0, corePoolSize=3, maxPoolSize=3 2024-11-20T17:59:37,268 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,268 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,269 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,269 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,269 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,269 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,46615,1732125544057-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T17:59:37,288 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T17:59:37,290 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,46615,1732125544057-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,290 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,290 INFO [RS:0;613581fcdc40:46615 {}] regionserver.Replication(171): 613581fcdc40,46615,1732125544057 started 2024-11-20T17:59:37,309 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T17:59:37,309 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1482): Serving as 613581fcdc40,46615,1732125544057, RpcServer on 613581fcdc40/172.17.0.2:46615, sessionid=0x1015966431a0001 2024-11-20T17:59:37,310 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T17:59:37,310 DEBUG [RS:0;613581fcdc40:46615 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,311 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,46615,1732125544057' 2024-11-20T17:59:37,311 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T17:59:37,312 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T17:59:37,312 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T17:59:37,312 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T17:59:37,312 DEBUG [RS:0;613581fcdc40:46615 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 613581fcdc40,46615,1732125544057 2024-11-20T17:59:37,313 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '613581fcdc40,46615,1732125544057' 2024-11-20T17:59:37,313 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T17:59:37,313 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T17:59:37,314 DEBUG [RS:0;613581fcdc40:46615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T17:59:37,314 INFO [RS:0;613581fcdc40:46615 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T17:59:37,314 INFO [RS:0;613581fcdc40:46615 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T17:59:37,356 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:38,356 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:39,357 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:40,358 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:41,358 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:42,359 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:43,360 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:44,361 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:45,361 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:46,362 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:47,363 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:48,364 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:49,364 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:50,365 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:51,366 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:52,366 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:53,367 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:54,368 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:55,369 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:56,370 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:57,371 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:58,371 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T17:59:59,372 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:00,373 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:01,374 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:02,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T18:00:02,374 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:03,375 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:04,375 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:05,376 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:06,377 WARN [613581fcdc40:35971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T18:00:07,320 INFO [RS:0;613581fcdc40:46615 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T18:00:07,323 INFO [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C46615%2C1732125544057, suffix=, logDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057, archiveDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/oldWALs, maxLogs=32 2024-11-20T18:00:07,343 DEBUG [RS:0;613581fcdc40:46615 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057/613581fcdc40%2C46615%2C1732125544057.1732125607326, exclude list is [], retry=0 2024-11-20T18:00:07,348 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34453,DS-aa4f7914-7084-4fbe-9372-838d0e479274,DISK] 2024-11-20T18:00:07,352 INFO [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057/613581fcdc40%2C46615%2C1732125544057.1732125607326 2024-11-20T18:00:07,354 DEBUG [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46027:46027)] 2024-11-20T18:00:07,379 DEBUG [613581fcdc40:35971 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T18:00:07,392 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:07,399 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,46615,1732125544057, state=OPENING 2024-11-20T18:00:07,477 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T18:00:07,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T18:00:07,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T18:00:07,489 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T18:00:07,489 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T18:00:07,490 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T18:00:07,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:07,669 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T18:00:07,673 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T18:00:07,683 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T18:00:07,683 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T18:00:07,684 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T18:00:07,688 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=613581fcdc40%2C46615%2C1732125544057.meta, suffix=.meta, logDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057, archiveDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/oldWALs, maxLogs=32 2024-11-20T18:00:07,712 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057/613581fcdc40%2C46615%2C1732125544057.meta.1732125607691.meta, exclude list is [], retry=0 2024-11-20T18:00:07,716 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34453,DS-aa4f7914-7084-4fbe-9372-838d0e479274,DISK] 2024-11-20T18:00:07,722 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/WALs/613581fcdc40,46615,1732125544057/613581fcdc40%2C46615%2C1732125544057.meta.1732125607691.meta 2024-11-20T18:00:07,722 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46027:46027)] 2024-11-20T18:00:07,722 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:07,724 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T18:00:07,727 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T18:00:07,731 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T18:00:07,736 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T18:00:07,736 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:07,737 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T18:00:07,737 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T18:00:07,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T18:00:07,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T18:00:07,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:07,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T18:00:07,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T18:00:07,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T18:00:07,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:07,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T18:00:07,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T18:00:07,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T18:00:07,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:07,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T18:00:07,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T18:00:07,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T18:00:07,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:07,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T18:00:07,752 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T18:00:07,753 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740 2024-11-20T18:00:07,756 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740 2024-11-20T18:00:07,758 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T18:00:07,758 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T18:00:07,759 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T18:00:07,762 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T18:00:07,764 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67139144, jitterRate=4.5120716094970703E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T18:00:07,764 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T18:00:07,766 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732125607737Writing region info on filesystem at 1732125607737Initializing all the Stores at 1732125607739 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125607739Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125607740 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125607740Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732125607740Cleaning up temporary data from old regions at 1732125607758 (+18 ms)Running coprocessor post-open hooks at 1732125607764 (+6 ms)Region opened successfully at 1732125607766 (+2 ms) 2024-11-20T18:00:07,773 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732125607660 2024-11-20T18:00:07,784 DEBUG [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T18:00:07,785 INFO [RS_OPEN_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T18:00:07,788 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:07,791 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 613581fcdc40,46615,1732125544057, state=OPEN 2024-11-20T18:00:07,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T18:00:07,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T18:00:07,852 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T18:00:07,852 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T18:00:07,853 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=613581fcdc40,46615,1732125544057 2024-11-20T18:00:07,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T18:00:07,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=613581fcdc40,46615,1732125544057 in 361 msec 2024-11-20T18:00:07,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T18:00:07,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.693 sec 2024-11-20T18:00:07,872 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T18:00:07,872 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T18:00:07,892 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T18:00:07,894 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,46615,1732125544057, seqNum=-1] 2024-11-20T18:00:07,921 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T18:00:07,924 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39307, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T18:00:07,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 0.993 sec 2024-11-20T18:00:07,951 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732125607950, completionTime=-1 2024-11-20T18:00:07,955 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T18:00:07,955 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T18:00:08,012 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-20T18:00:08,013 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732125668013 2024-11-20T18:00:08,013 INFO [master/613581fcdc40:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732125728013 2024-11-20T18:00:08,013 INFO [master/613581fcdc40:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 57 msec 2024-11-20T18:00:08,015 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,015 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,015 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,017 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-613581fcdc40:35971, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,018 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,019 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,026 DEBUG [master/613581fcdc40:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T18:00:08,055 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.804sec 2024-11-20T18:00:08,057 INFO [master/613581fcdc40:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T18:00:08,058 INFO [master/613581fcdc40:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T18:00:08,059 INFO [master/613581fcdc40:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T18:00:08,060 INFO [master/613581fcdc40:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T18:00:08,060 INFO [master/613581fcdc40:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T18:00:08,061 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T18:00:08,062 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T18:00:08,070 DEBUG [master/613581fcdc40:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T18:00:08,071 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T18:00:08,072 INFO [master/613581fcdc40:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=613581fcdc40,35971,1732125543153-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T18:00:08,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72819edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,132 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T18:00:08,132 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T18:00:08,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,35971,-1 for getting cluster id 2024-11-20T18:00:08,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T18:00:08,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e4bcc2a-a205-44fb-ba82-aeb59a139b49' 2024-11-20T18:00:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T18:00:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e4bcc2a-a205-44fb-ba82-aeb59a139b49" 2024-11-20T18:00:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4411660c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,35971,-1] 2024-11-20T18:00:08,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T18:00:08,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:00:08,158 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T18:00:08,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46433c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T18:00:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,46615,1732125544057, seqNum=-1] 2024-11-20T18:00:08,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T18:00:08,173 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T18:00:08,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=613581fcdc40,35971,1732125543153 2024-11-20T18:00:08,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e24924e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 613581fcdc40,35971,-1 for getting cluster id 2024-11-20T18:00:08,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T18:00:08,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e4bcc2a-a205-44fb-ba82-aeb59a139b49' 2024-11-20T18:00:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T18:00:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e4bcc2a-a205-44fb-ba82-aeb59a139b49" 2024-11-20T18:00:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee2f7f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [613581fcdc40,35971,-1] 2024-11-20T18:00:08,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T18:00:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:00:08,211 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T18:00:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a9f507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T18:00:08,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=8325 2024-11-20T18:00:08,242 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T18:00:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 613581fcdc40,35971,1732125543153 2024-11-20T18:00:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76c5f21 2024-11-20T18:00:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T18:00:08,258 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T18:00:08,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T18:00:08,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:08,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-20T18:00:08,279 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:08,282 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T18:00:08,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741835_1011 (size=428) 2024-11-20T18:00:08,708 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dcd532c37d83a3d6feb163c60d6720fa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:08,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741836_1012 (size=79) 2024-11-20T18:00:09,120 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:09,120 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing dcd532c37d83a3d6feb163c60d6720fa, disabling compactions & flushes 2024-11-20T18:00:09,120 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,120 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,121 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. after waiting 0 ms 2024-11-20T18:00:09,121 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,121 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,121 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for dcd532c37d83a3d6feb163c60d6720fa: Waiting for close lock at 1732125609120Disabling compacts and flushes for region at 1732125609120Disabling writes for close at 1732125609121 (+1 ms)Writing region close event to WAL at 1732125609121Closed at 1732125609121 2024-11-20T18:00:09,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:09,128 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732125609124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125609124"}]},"ts":"1732125609124"} 2024-11-20T18:00:09,134 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:09,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:09,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125609136"}]},"ts":"1732125609136"} 2024-11-20T18:00:09,145 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-20T18:00:09,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=dcd532c37d83a3d6feb163c60d6720fa, ASSIGN}] 2024-11-20T18:00:09,150 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=dcd532c37d83a3d6feb163c60d6720fa, ASSIGN 2024-11-20T18:00:09,152 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=dcd532c37d83a3d6feb163c60d6720fa, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:09,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcd532c37d83a3d6feb163c60d6720fa, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T18:00:09,310 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=dcd532c37d83a3d6feb163c60d6720fa, ASSIGN because future has completed 2024-11-20T18:00:09,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcd532c37d83a3d6feb163c60d6720fa, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:09,473 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,473 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dcd532c37d83a3d6feb163c60d6720fa, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:09,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:09,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,474 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,477 INFO [StoreOpener-dcd532c37d83a3d6feb163c60d6720fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,480 INFO [StoreOpener-dcd532c37d83a3d6feb163c60d6720fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcd532c37d83a3d6feb163c60d6720fa columnFamilyName testFamily 2024-11-20T18:00:09,480 DEBUG [StoreOpener-dcd532c37d83a3d6feb163c60d6720fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:09,481 INFO [StoreOpener-dcd532c37d83a3d6feb163c60d6720fa-1 {}] regionserver.HStore(327): Store=dcd532c37d83a3d6feb163c60d6720fa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:09,481 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,483 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,484 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,484 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,484 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,488 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,493 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:09,494 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dcd532c37d83a3d6feb163c60d6720fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70851737, jitterRate=0.05577315390110016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:09,494 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:00:09,496 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dcd532c37d83a3d6feb163c60d6720fa: Running coprocessor pre-open hook at 1732125609475Writing region info on filesystem at 1732125609475Initializing all the Stores at 1732125609477 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125609477Cleaning up temporary data from old regions at 1732125609484 (+7 ms)Running coprocessor post-open hooks at 1732125609494 (+10 ms)Region opened successfully at 1732125609495 (+1 ms) 2024-11-20T18:00:09,498 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., pid=6, masterSystemTime=1732125609466 2024-11-20T18:00:09,502 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,502 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:00:09,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcd532c37d83a3d6feb163c60d6720fa, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:09,509 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcd532c37d83a3d6feb163c60d6720fa, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:09,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T18:00:09,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dcd532c37d83a3d6feb163c60d6720fa, server=613581fcdc40,46615,1732125544057 in 200 msec 2024-11-20T18:00:09,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T18:00:09,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=dcd532c37d83a3d6feb163c60d6720fa, ASSIGN in 369 msec 2024-11-20T18:00:09,524 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:09,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125609524"}]},"ts":"1732125609524"} 2024-11-20T18:00:09,528 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-20T18:00:09,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:09,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2620 sec 2024-11-20T18:00:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T18:00:11,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-20T18:00:11,317 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-20T18:00:11,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:11,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-20T18:00:11,324 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:11,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-20T18:00:11,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T18:00:11,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=613581fcdc40,46615,1732125544057, seqNum=-1] 2024-11-20T18:00:11,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T18:00:11,332 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T18:00:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:11,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:11,357 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-20T18:00:11,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T18:00:11,366 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T18:00:11,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T18:00:11,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8139664020417282040 2024-11-20T18:00:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125614367, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:11,376 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T18:00:11,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T18:00:11,379 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:11,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:11,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8139664020417282038 2024-11-20T18:00:11,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125614380, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:11,385 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:11,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8139664020417282037 2024-11-20T18:00:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125614384, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:11,389 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:11,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8139664020417282036 2024-11-20T18:00:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125614388, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:11,393 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:11,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_124898416_22 at /127.0.0.1:58688 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 269) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8287 (was 8325) 2024-11-20T18:00:11,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=8287 2024-11-20T18:00:11,415 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:11,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-20T18:00:11,420 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:11,420 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:11,420 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-20T18:00:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T18:00:11,422 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:11,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741837_1013 (size=414) 2024-11-20T18:00:11,835 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef28d09bb8b7b63f2ec04d794276a93e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:11,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741838_1014 (size=65) 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ef28d09bb8b7b63f2ec04d794276a93e, disabling compactions & flushes 2024-11-20T18:00:12,246 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. after waiting 0 ms 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,246 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef28d09bb8b7b63f2ec04d794276a93e: Waiting for close lock at 1732125612246Disabling compacts and flushes for region at 1732125612246Disabling writes for close at 1732125612246Writing region close event to WAL at 1732125612246Closed at 1732125612246 2024-11-20T18:00:12,248 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:12,249 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732125612248"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125612248"}]},"ts":"1732125612248"} 2024-11-20T18:00:12,252 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:12,254 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:12,254 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125612254"}]},"ts":"1732125612254"} 2024-11-20T18:00:12,258 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T18:00:12,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ef28d09bb8b7b63f2ec04d794276a93e, ASSIGN}] 2024-11-20T18:00:12,260 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ef28d09bb8b7b63f2ec04d794276a93e, ASSIGN 2024-11-20T18:00:12,262 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ef28d09bb8b7b63f2ec04d794276a93e, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:12,414 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ef28d09bb8b7b63f2ec04d794276a93e, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:12,423 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ef28d09bb8b7b63f2ec04d794276a93e, ASSIGN because future has completed 2024-11-20T18:00:12,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef28d09bb8b7b63f2ec04d794276a93e, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T18:00:12,592 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ef28d09bb8b7b63f2ec04d794276a93e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:12,593 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:12,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,597 INFO [StoreOpener-ef28d09bb8b7b63f2ec04d794276a93e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,599 INFO [StoreOpener-ef28d09bb8b7b63f2ec04d794276a93e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef28d09bb8b7b63f2ec04d794276a93e columnFamilyName testFamily 2024-11-20T18:00:12,599 DEBUG [StoreOpener-ef28d09bb8b7b63f2ec04d794276a93e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:12,599 INFO [StoreOpener-ef28d09bb8b7b63f2ec04d794276a93e-1 {}] regionserver.HStore(327): Store=ef28d09bb8b7b63f2ec04d794276a93e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:12,600 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,608 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:12,609 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened ef28d09bb8b7b63f2ec04d794276a93e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61755174, jitterRate=-0.07977619767189026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:12,609 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:00:12,610 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ef28d09bb8b7b63f2ec04d794276a93e: Running coprocessor pre-open hook at 1732125612594Writing region info on filesystem at 1732125612594Initializing all the Stores at 1732125612596 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125612596Cleaning up temporary data from old regions at 1732125612602 (+6 ms)Running coprocessor post-open hooks at 1732125612609 (+7 ms)Region opened successfully at 1732125612610 (+1 ms) 2024-11-20T18:00:12,611 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., pid=9, masterSystemTime=1732125612580 2024-11-20T18:00:12,615 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,615 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:00:12,616 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ef28d09bb8b7b63f2ec04d794276a93e, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:12,619 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef28d09bb8b7b63f2ec04d794276a93e, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:12,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T18:00:12,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ef28d09bb8b7b63f2ec04d794276a93e, server=613581fcdc40,46615,1732125544057 in 197 msec 2024-11-20T18:00:12,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T18:00:12,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ef28d09bb8b7b63f2ec04d794276a93e, ASSIGN in 367 msec 2024-11-20T18:00:12,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:12,631 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125612631"}]},"ts":"1732125612631"} 2024-11-20T18:00:12,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T18:00:12,635 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:12,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2200 sec 2024-11-20T18:00:13,278 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:00:13,307 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-20T18:00:13,309 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T18:00:13,310 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-20T18:00:13,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T18:00:13,577 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:13,579 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T18:00:13,579 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T18:00:13,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-20T18:00:13,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:13,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-20T18:00:13,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T18:00:14,457 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-20T18:00:14,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T18:00:14,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:14,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T18:00:14,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:14,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-20T18:00:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:14,478 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:14,478 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T18:00:14,482 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T18:00:14,484 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T18:00:20,494 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-20T18:00:20,494 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-20T18:00:20,495 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:00:20,508 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=296 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=9328 (was 8287) - AvailableMemoryMB LEAK? - 2024-11-20T18:00:20,517 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=296, ProcessCount=11, AvailableMemoryMB=9325 2024-11-20T18:00:20,520 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:20,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-20T18:00:20,524 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:20,524 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:20,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-20T18:00:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T18:00:20,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:20,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741839_1015 (size=412) 2024-11-20T18:00:20,939 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b87905cc81e3d78383b58552be822048, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:20,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741840_1016 (size=63) 2024-11-20T18:00:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-20T18:00:21,351 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:21,351 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing b87905cc81e3d78383b58552be822048, disabling compactions & flushes 2024-11-20T18:00:21,351 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,351 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,351 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. after waiting 0 ms 2024-11-20T18:00:21,352 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,352 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,352 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b87905cc81e3d78383b58552be822048: Waiting for close lock at 1732125621351Disabling compacts and flushes for region at 1732125621351Disabling writes for close at 1732125621351Writing region close event to WAL at 1732125621352 (+1 ms)Closed at 1732125621352 2024-11-20T18:00:21,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:21,354 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732125621354"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125621354"}]},"ts":"1732125621354"} 2024-11-20T18:00:21,357 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:21,359 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:21,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125621359"}]},"ts":"1732125621359"} 2024-11-20T18:00:21,362 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T18:00:21,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=b87905cc81e3d78383b58552be822048, ASSIGN}] 2024-11-20T18:00:21,365 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=b87905cc81e3d78383b58552be822048, ASSIGN 2024-11-20T18:00:21,367 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=b87905cc81e3d78383b58552be822048, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:21,517 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b87905cc81e3d78383b58552be822048, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:21,522 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=b87905cc81e3d78383b58552be822048, ASSIGN because future has completed 2024-11-20T18:00:21,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b87905cc81e3d78383b58552be822048, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T18:00:21,683 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => b87905cc81e3d78383b58552be822048, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:21,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:21,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,687 INFO [StoreOpener-b87905cc81e3d78383b58552be822048-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,689 INFO [StoreOpener-b87905cc81e3d78383b58552be822048-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b87905cc81e3d78383b58552be822048 columnFamilyName testFamily 2024-11-20T18:00:21,690 DEBUG [StoreOpener-b87905cc81e3d78383b58552be822048-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:21,691 INFO [StoreOpener-b87905cc81e3d78383b58552be822048-1 {}] regionserver.HStore(327): Store=b87905cc81e3d78383b58552be822048/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:21,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,693 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:21,703 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened b87905cc81e3d78383b58552be822048; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65405867, jitterRate=-0.025376632809638977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:21,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b87905cc81e3d78383b58552be822048 2024-11-20T18:00:21,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for b87905cc81e3d78383b58552be822048: Running coprocessor pre-open hook at 1732125621685Writing region info on filesystem at 1732125621685Initializing all the Stores at 1732125621686 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125621686Cleaning up temporary data from old regions at 1732125621694 (+8 ms)Running coprocessor post-open hooks at 1732125621703 (+9 ms)Region opened successfully at 1732125621704 (+1 ms) 2024-11-20T18:00:21,706 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048., pid=12, masterSystemTime=1732125621677 2024-11-20T18:00:21,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,709 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:00:21,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b87905cc81e3d78383b58552be822048, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:21,714 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b87905cc81e3d78383b58552be822048, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:21,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T18:00:21,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b87905cc81e3d78383b58552be822048, server=613581fcdc40,46615,1732125544057 in 194 msec 2024-11-20T18:00:21,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-20T18:00:21,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=b87905cc81e3d78383b58552be822048, ASSIGN in 359 msec 2024-11-20T18:00:21,727 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:21,728 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125621727"}]},"ts":"1732125621727"} 2024-11-20T18:00:21,730 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T18:00:21,732 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:21,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2120 sec 2024-11-20T18:00:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-20T18:00:23,559 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-20T18:00:23,560 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T18:00:23,560 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:23,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T18:00:23,571 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:23,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-20T18:00:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:23,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-20T18:00:23,576 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:23,586 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:23,586 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T18:00:23,590 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T18:00:23,593 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T18:00:23,594 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T18:00:23,595 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T18:00:25,311 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-20T18:00:27,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:00:29,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282027' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T18:00:29,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51634 deadline: 1732125635602, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282027' does not exist 2024-11-20T18:00:29,609 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 613581fcdc40,46615,1732125544057 for closing scanner id = -8139664020417282027 for b87905cc81e3d78383b58552be822048 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282027' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T18:00:29,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 296), ProcessCount=11 (was 11), AvailableMemoryMB=8986 (was 9325) 2024-11-20T18:00:29,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=8985 2024-11-20T18:00:29,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-20T18:00:29,627 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:29,628 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:29,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-20T18:00:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T18:00:29,630 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:29,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741841_1017 (size=423) 2024-11-20T18:00:30,045 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7bacbf91cca191c9255ffd957a757d2, NAME => 'testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741842_1018 (size=74) 2024-11-20T18:00:30,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing a7bacbf91cca191c9255ffd957a757d2, disabling compactions & flushes 2024-11-20T18:00:30,460 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. after waiting 0 ms 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,460 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7bacbf91cca191c9255ffd957a757d2: Waiting for close lock at 1732125630460Disabling compacts and flushes for region at 1732125630460Disabling writes for close at 1732125630460Writing region close event to WAL at 1732125630460Closed at 1732125630460 2024-11-20T18:00:30,462 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:30,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732125630462"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125630462"}]},"ts":"1732125630462"} 2024-11-20T18:00:30,465 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:30,466 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:30,466 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125630466"}]},"ts":"1732125630466"} 2024-11-20T18:00:30,469 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-20T18:00:30,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a7bacbf91cca191c9255ffd957a757d2, ASSIGN}] 2024-11-20T18:00:30,471 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a7bacbf91cca191c9255ffd957a757d2, ASSIGN 2024-11-20T18:00:30,473 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a7bacbf91cca191c9255ffd957a757d2, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:30,624 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a7bacbf91cca191c9255ffd957a757d2, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:30,629 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a7bacbf91cca191c9255ffd957a757d2, ASSIGN because future has completed 2024-11-20T18:00:30,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7bacbf91cca191c9255ffd957a757d2, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T18:00:30,792 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,792 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => a7bacbf91cca191c9255ffd957a757d2, NAME => 'testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:30,793 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,793 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:30,793 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,793 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,795 INFO [StoreOpener-a7bacbf91cca191c9255ffd957a757d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,798 INFO [StoreOpener-a7bacbf91cca191c9255ffd957a757d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7bacbf91cca191c9255ffd957a757d2 columnFamilyName testFamily 2024-11-20T18:00:30,798 DEBUG [StoreOpener-a7bacbf91cca191c9255ffd957a757d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:30,799 INFO [StoreOpener-a7bacbf91cca191c9255ffd957a757d2-1 {}] regionserver.HStore(327): Store=a7bacbf91cca191c9255ffd957a757d2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:30,799 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,801 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,802 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,803 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,803 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,806 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,810 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:30,810 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened a7bacbf91cca191c9255ffd957a757d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72783790, jitterRate=0.0845629870891571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:30,811 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:00:30,811 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for a7bacbf91cca191c9255ffd957a757d2: Running coprocessor pre-open hook at 1732125630793Writing region info on filesystem at 1732125630793Initializing all the Stores at 1732125630795 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125630795Cleaning up temporary data from old regions at 1732125630803 (+8 ms)Running coprocessor post-open hooks at 1732125630811 (+8 ms)Region opened successfully at 1732125630811 2024-11-20T18:00:30,813 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2., pid=15, masterSystemTime=1732125630786 2024-11-20T18:00:30,816 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,816 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:00:30,817 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a7bacbf91cca191c9255ffd957a757d2, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:30,820 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7bacbf91cca191c9255ffd957a757d2, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:30,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-20T18:00:30,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a7bacbf91cca191c9255ffd957a757d2, server=613581fcdc40,46615,1732125544057 in 191 msec 2024-11-20T18:00:30,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T18:00:30,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a7bacbf91cca191c9255ffd957a757d2, ASSIGN in 355 msec 2024-11-20T18:00:30,829 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:30,829 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125630829"}]},"ts":"1732125630829"} 2024-11-20T18:00:30,832 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-20T18:00:30,833 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:30,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2100 sec 2024-11-20T18:00:31,313 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-20T18:00:32,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T18:00:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T18:00:32,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-20T18:00:32,657 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-20T18:00:32,658 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:32,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-20T18:00:32,667 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:32,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-20T18:00:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:32,681 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:32,682 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-20T18:00:32,685 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-20T18:00:32,687 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-20T18:00:32,689 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-20T18:00:32,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8139664020417282023 2024-11-20T18:00:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125635688, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:32,693 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-20T18:00:32,694 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-20T18:00:32,695 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:32,697 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:32,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8139664020417282021 2024-11-20T18:00:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125635696, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:32,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:32,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8139664020417282020 2024-11-20T18:00:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125635700, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:32,705 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:32,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8139664020417282019 2024-11-20T18:00:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:51634 deadline: 1732125635704, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-20T18:00:32,709 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-20T18:00:32,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_124898416_22 at /127.0.0.1:49754 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 265), ProcessCount=11 (was 11), AvailableMemoryMB=8957 (was 8985) 2024-11-20T18:00:32,727 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=8957 2024-11-20T18:00:32,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-20T18:00:32,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:32,733 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:32,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-20T18:00:32,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T18:00:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741843_1019 (size=418) 2024-11-20T18:00:33,149 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 35c5b501588438fab9b324c90dbb0d72, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:33,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741844_1020 (size=69) 2024-11-20T18:00:33,562 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:33,562 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 35c5b501588438fab9b324c90dbb0d72, disabling compactions & flushes 2024-11-20T18:00:33,562 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,562 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,562 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. after waiting 0 ms 2024-11-20T18:00:33,563 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,563 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,563 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 35c5b501588438fab9b324c90dbb0d72: Waiting for close lock at 1732125633562Disabling compacts and flushes for region at 1732125633562Disabling writes for close at 1732125633563 (+1 ms)Writing region close event to WAL at 1732125633563Closed at 1732125633563 2024-11-20T18:00:33,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:33,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732125633565"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125633565"}]},"ts":"1732125633565"} 2024-11-20T18:00:33,570 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:33,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:33,572 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125633572"}]},"ts":"1732125633572"} 2024-11-20T18:00:33,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-20T18:00:33,576 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-20T18:00:33,576 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:33,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=35c5b501588438fab9b324c90dbb0d72, ASSIGN}] 2024-11-20T18:00:33,579 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=35c5b501588438fab9b324c90dbb0d72, ASSIGN 2024-11-20T18:00:33,580 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=35c5b501588438fab9b324c90dbb0d72, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:33,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=35c5b501588438fab9b324c90dbb0d72, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:33,734 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=35c5b501588438fab9b324c90dbb0d72, ASSIGN because future has completed 2024-11-20T18:00:33,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35c5b501588438fab9b324c90dbb0d72, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T18:00:33,893 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,894 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 35c5b501588438fab9b324c90dbb0d72, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:33,894 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,894 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:33,894 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,894 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,896 INFO [StoreOpener-35c5b501588438fab9b324c90dbb0d72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,898 INFO [StoreOpener-35c5b501588438fab9b324c90dbb0d72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35c5b501588438fab9b324c90dbb0d72 columnFamilyName testFamily 2024-11-20T18:00:33,898 DEBUG [StoreOpener-35c5b501588438fab9b324c90dbb0d72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:33,899 INFO [StoreOpener-35c5b501588438fab9b324c90dbb0d72-1 {}] regionserver.HStore(327): Store=35c5b501588438fab9b324c90dbb0d72/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:33,899 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,900 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,901 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,901 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,901 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,904 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,907 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:33,908 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 35c5b501588438fab9b324c90dbb0d72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70214886, jitterRate=0.046283334493637085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:33,908 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:00:33,908 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 35c5b501588438fab9b324c90dbb0d72: Running coprocessor pre-open hook at 1732125633895Writing region info on filesystem at 1732125633895Initializing all the Stores at 1732125633896 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125633896Cleaning up temporary data from old regions at 1732125633901 (+5 ms)Running coprocessor post-open hooks at 1732125633908 (+7 ms)Region opened successfully at 1732125633908 2024-11-20T18:00:33,909 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., pid=18, masterSystemTime=1732125633888 2024-11-20T18:00:33,912 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,913 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:00:33,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=35c5b501588438fab9b324c90dbb0d72, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:33,917 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35c5b501588438fab9b324c90dbb0d72, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:33,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T18:00:33,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 35c5b501588438fab9b324c90dbb0d72, server=613581fcdc40,46615,1732125544057 in 184 msec 2024-11-20T18:00:33,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-20T18:00:33,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=35c5b501588438fab9b324c90dbb0d72, ASSIGN in 344 msec 2024-11-20T18:00:33,925 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:33,926 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125633925"}]},"ts":"1732125633925"} 2024-11-20T18:00:33,928 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-20T18:00:33,929 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:33,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1990 sec 2024-11-20T18:00:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-20T18:00:35,767 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-20T18:00:35,767 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-20T18:00:35,767 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:35,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-20T18:00:35,776 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:35,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-20T18:00:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:35,789 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:35,789 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-20T18:00:35,792 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T18:00:35,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T18:00:36,295 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:00:36,796 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T18:00:36,796 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-20T18:00:36,797 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:00:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-20T18:00:37,314 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-20T18:00:37,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:37,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T18:00:38,796 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=981ms 2024-11-20T18:00:38,796 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=981ms 2024-11-20T18:00:38,796 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:00:38,810 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 265), ProcessCount=11 (was 11), AvailableMemoryMB=8911 (was 8957) 2024-11-20T18:00:38,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=8911 2024-11-20T18:00:38,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-20T18:00:38,823 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:38,823 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:38,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-20T18:00:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T18:00:38,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741845_1021 (size=416) 2024-11-20T18:00:38,838 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e0d943f6ccd4d61aee4e842944711b2b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:38,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741846_1022 (size=67) 2024-11-20T18:00:39,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:39,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing e0d943f6ccd4d61aee4e842944711b2b, disabling compactions & flushes 2024-11-20T18:00:39,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. after waiting 0 ms 2024-11-20T18:00:39,247 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,247 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,248 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e0d943f6ccd4d61aee4e842944711b2b: Waiting for close lock at 1732125639247Disabling compacts and flushes for region at 1732125639247Disabling writes for close at 1732125639247Writing region close event to WAL at 1732125639247Closed at 1732125639247 2024-11-20T18:00:39,250 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:39,250 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732125639250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125639250"}]},"ts":"1732125639250"} 2024-11-20T18:00:39,253 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:39,255 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:39,255 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125639255"}]},"ts":"1732125639255"} 2024-11-20T18:00:39,258 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-20T18:00:39,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e0d943f6ccd4d61aee4e842944711b2b, ASSIGN}] 2024-11-20T18:00:39,260 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e0d943f6ccd4d61aee4e842944711b2b, ASSIGN 2024-11-20T18:00:39,262 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e0d943f6ccd4d61aee4e842944711b2b, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-20T18:00:39,413 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e0d943f6ccd4d61aee4e842944711b2b, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:39,416 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e0d943f6ccd4d61aee4e842944711b2b, ASSIGN because future has completed 2024-11-20T18:00:39,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0d943f6ccd4d61aee4e842944711b2b, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:39,579 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,579 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e0d943f6ccd4d61aee4e842944711b2b, NAME => 'testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:39,580 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,580 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:39,580 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,580 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,582 INFO [StoreOpener-e0d943f6ccd4d61aee4e842944711b2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,584 INFO [StoreOpener-e0d943f6ccd4d61aee4e842944711b2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0d943f6ccd4d61aee4e842944711b2b columnFamilyName testFamily 2024-11-20T18:00:39,585 DEBUG [StoreOpener-e0d943f6ccd4d61aee4e842944711b2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:39,585 INFO [StoreOpener-e0d943f6ccd4d61aee4e842944711b2b-1 {}] regionserver.HStore(327): Store=e0d943f6ccd4d61aee4e842944711b2b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:39,586 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,587 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,587 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,588 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,588 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,590 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,593 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:39,594 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e0d943f6ccd4d61aee4e842944711b2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74201250, jitterRate=0.1056847870349884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:39,594 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:00:39,596 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e0d943f6ccd4d61aee4e842944711b2b: Running coprocessor pre-open hook at 1732125639580Writing region info on filesystem at 1732125639580Initializing all the Stores at 1732125639581 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125639581Cleaning up temporary data from old regions at 1732125639588 (+7 ms)Running coprocessor post-open hooks at 1732125639594 (+6 ms)Region opened successfully at 1732125639596 (+2 ms) 2024-11-20T18:00:39,597 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., pid=21, masterSystemTime=1732125639570 2024-11-20T18:00:39,600 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,600 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:00:39,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e0d943f6ccd4d61aee4e842944711b2b, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:39,605 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0d943f6ccd4d61aee4e842944711b2b, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:39,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-20T18:00:39,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e0d943f6ccd4d61aee4e842944711b2b, server=613581fcdc40,46615,1732125544057 in 190 msec 2024-11-20T18:00:39,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T18:00:39,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e0d943f6ccd4d61aee4e842944711b2b, ASSIGN in 352 msec 2024-11-20T18:00:39,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:39,616 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125639615"}]},"ts":"1732125639615"} 2024-11-20T18:00:39,618 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-20T18:00:39,620 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:39,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 800 msec 2024-11-20T18:00:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T18:00:39,837 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-20T18:00:39,837 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-20T18:00:39,837 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:39,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-20T18:00:39,846 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:39,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-20T18:00:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:39,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:39,862 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-20T18:00:39,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T18:00:39,867 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T18:00:39,868 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T18:00:39,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T18:00:42,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282008' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T18:00:42,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51634 deadline: 1732125643878, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282008' does not exist 2024-11-20T18:00:42,879 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 613581fcdc40,46615,1732125544057 for closing scanner id = -8139664020417282008 for e0d943f6ccd4d61aee4e842944711b2b of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282008' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T18:00:42,883 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_124898416_22 at /127.0.0.1:56748 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=8804 (was 8911) 2024-11-20T18:00:42,891 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=8803 2024-11-20T18:00:42,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-20T18:00:42,896 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:42,896 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:42,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-20T18:00:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T18:00:42,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:42,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741847_1023 (size=411) 2024-11-20T18:00:43,308 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c970d752579eca03ef286dd82dac0e9f, NAME => 'testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:43,317 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-20T18:00:43,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741848_1024 (size=62) 2024-11-20T18:00:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-20T18:00:43,577 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-20T18:00:43,577 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:43,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-20T18:00:43,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:43,720 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:43,720 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing c970d752579eca03ef286dd82dac0e9f, disabling compactions & flushes 2024-11-20T18:00:43,720 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:43,720 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:43,721 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. after waiting 0 ms 2024-11-20T18:00:43,721 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:43,721 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:43,721 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for c970d752579eca03ef286dd82dac0e9f: Waiting for close lock at 1732125643720Disabling compacts and flushes for region at 1732125643720Disabling writes for close at 1732125643721 (+1 ms)Writing region close event to WAL at 1732125643721Closed at 1732125643721 2024-11-20T18:00:43,723 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:43,723 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732125643723"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125643723"}]},"ts":"1732125643723"} 2024-11-20T18:00:43,726 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:43,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:43,728 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125643727"}]},"ts":"1732125643727"} 2024-11-20T18:00:43,730 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T18:00:43,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c970d752579eca03ef286dd82dac0e9f, ASSIGN}] 2024-11-20T18:00:43,732 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c970d752579eca03ef286dd82dac0e9f, ASSIGN 2024-11-20T18:00:43,734 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c970d752579eca03ef286dd82dac0e9f, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:43,885 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c970d752579eca03ef286dd82dac0e9f, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:43,889 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c970d752579eca03ef286dd82dac0e9f, ASSIGN because future has completed 2024-11-20T18:00:43,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c970d752579eca03ef286dd82dac0e9f, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T18:00:44,051 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:44,051 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => c970d752579eca03ef286dd82dac0e9f, NAME => 'testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:44,052 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,052 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:44,052 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,052 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,055 INFO [StoreOpener-c970d752579eca03ef286dd82dac0e9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,058 INFO [StoreOpener-c970d752579eca03ef286dd82dac0e9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c970d752579eca03ef286dd82dac0e9f columnFamilyName testFamily 2024-11-20T18:00:44,059 DEBUG [StoreOpener-c970d752579eca03ef286dd82dac0e9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:44,059 INFO [StoreOpener-c970d752579eca03ef286dd82dac0e9f-1 {}] regionserver.HStore(327): Store=c970d752579eca03ef286dd82dac0e9f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:44,060 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,061 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,061 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,062 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,062 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,064 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,067 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:44,067 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened c970d752579eca03ef286dd82dac0e9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69185753, jitterRate=0.030948057770729065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:44,067 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:00:44,068 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for c970d752579eca03ef286dd82dac0e9f: Running coprocessor pre-open hook at 1732125644053Writing region info on filesystem at 1732125644053Initializing all the Stores at 1732125644055 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125644055Cleaning up temporary data from old regions at 1732125644062 (+7 ms)Running coprocessor post-open hooks at 1732125644067 (+5 ms)Region opened successfully at 1732125644068 (+1 ms) 2024-11-20T18:00:44,069 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f., pid=24, masterSystemTime=1732125644045 2024-11-20T18:00:44,071 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:44,072 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:00:44,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c970d752579eca03ef286dd82dac0e9f, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:44,075 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c970d752579eca03ef286dd82dac0e9f, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:44,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T18:00:44,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c970d752579eca03ef286dd82dac0e9f, server=613581fcdc40,46615,1732125544057 in 186 msec 2024-11-20T18:00:44,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-20T18:00:44,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=c970d752579eca03ef286dd82dac0e9f, ASSIGN in 348 msec 2024-11-20T18:00:44,083 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:44,084 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125644083"}]},"ts":"1732125644083"} 2024-11-20T18:00:44,086 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T18:00:44,088 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:44,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1950 sec 2024-11-20T18:00:45,082 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:00:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-20T18:00:45,916 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-20T18:00:45,916 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T18:00:45,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:45,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T18:00:45,925 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:45,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-20T18:00:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:45,939 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:45,940 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T18:00:45,943 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T18:00:45,946 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T18:00:45,948 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T18:00:45,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-20T18:00:48,586 INFO [master/613581fcdc40:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T18:00:48,586 INFO [master/613581fcdc40:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T18:00:48,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282004' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T18:00:48,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51634 deadline: 1732125649956, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282004' does not exist 2024-11-20T18:00:48,960 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 613581fcdc40,46615,1732125544057 for closing scanner id = -8139664020417282004 for c970d752579eca03ef286dd82dac0e9f of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417282004' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T18:00:48,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=8687 (was 8803) 2024-11-20T18:00:48,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=8686 2024-11-20T18:00:48,977 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:48,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T18:00:48,980 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:48,980 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:48,980 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-20T18:00:48,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T18:00:48,981 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741849_1025 (size=419) 2024-11-20T18:00:49,318 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-20T18:00:49,393 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe119899672709659768ffe936fc77d3, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:49,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741850_1026 (size=70) 2024-11-20T18:00:49,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-20T18:00:49,804 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:49,804 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing fe119899672709659768ffe936fc77d3, disabling compactions & flushes 2024-11-20T18:00:49,804 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:49,804 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:49,805 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. after waiting 0 ms 2024-11-20T18:00:49,805 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:49,805 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:49,805 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe119899672709659768ffe936fc77d3: Waiting for close lock at 1732125649804Disabling compacts and flushes for region at 1732125649804Disabling writes for close at 1732125649805 (+1 ms)Writing region close event to WAL at 1732125649805Closed at 1732125649805 2024-11-20T18:00:49,808 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:49,808 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732125649808"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125649808"}]},"ts":"1732125649808"} 2024-11-20T18:00:49,813 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:49,815 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:49,816 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125649816"}]},"ts":"1732125649816"} 2024-11-20T18:00:49,819 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T18:00:49,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=fe119899672709659768ffe936fc77d3, ASSIGN}] 2024-11-20T18:00:49,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=fe119899672709659768ffe936fc77d3, ASSIGN 2024-11-20T18:00:49,822 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=fe119899672709659768ffe936fc77d3, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:49,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fe119899672709659768ffe936fc77d3, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:49,976 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=fe119899672709659768ffe936fc77d3, ASSIGN because future has completed 2024-11-20T18:00:49,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe119899672709659768ffe936fc77d3, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T18:00:50,142 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:50,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => fe119899672709659768ffe936fc77d3, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:50,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:50,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,145 INFO [StoreOpener-fe119899672709659768ffe936fc77d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,147 INFO [StoreOpener-fe119899672709659768ffe936fc77d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe119899672709659768ffe936fc77d3 columnFamilyName testFamily 2024-11-20T18:00:50,147 DEBUG [StoreOpener-fe119899672709659768ffe936fc77d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:50,148 INFO [StoreOpener-fe119899672709659768ffe936fc77d3-1 {}] regionserver.HStore(327): Store=fe119899672709659768ffe936fc77d3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:50,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,149 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,150 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,151 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,151 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,157 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:50,158 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened fe119899672709659768ffe936fc77d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74776014, jitterRate=0.11424943804740906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:50,158 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe119899672709659768ffe936fc77d3 2024-11-20T18:00:50,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for fe119899672709659768ffe936fc77d3: Running coprocessor pre-open hook at 1732125650143Writing region info on filesystem at 1732125650143Initializing all the Stores at 1732125650144 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125650144Cleaning up temporary data from old regions at 1732125650151 (+7 ms)Running coprocessor post-open hooks at 1732125650158 (+7 ms)Region opened successfully at 1732125650159 (+1 ms) 2024-11-20T18:00:50,160 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., pid=27, masterSystemTime=1732125650131 2024-11-20T18:00:50,163 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:50,163 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:00:50,164 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fe119899672709659768ffe936fc77d3, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:50,166 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe119899672709659768ffe936fc77d3, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:50,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T18:00:50,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure fe119899672709659768ffe936fc77d3, server=613581fcdc40,46615,1732125544057 in 192 msec 2024-11-20T18:00:50,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T18:00:50,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=fe119899672709659768ffe936fc77d3, ASSIGN in 351 msec 2024-11-20T18:00:50,174 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:50,174 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125650174"}]},"ts":"1732125650174"} 2024-11-20T18:00:50,176 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T18:00:50,177 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:50,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2000 sec 2024-11-20T18:00:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T18:00:52,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-20T18:00:52,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T18:00:52,017 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:52,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T18:00:52,024 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:00:52,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T18:00:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:00:52,036 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:00:52,036 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T18:00:52,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T18:00:52,040 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-20T18:00:53,577 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-20T18:00:53,577 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:53,580 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-20T18:00:53,580 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T18:00:55,673 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:00:55,697 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-20T18:00:58,045 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-20T18:00:58,045 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6006ms, rpcTimeout=5999ms 2024-11-20T18:00:58,045 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:00:58,056 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=8579 (was 8686) 2024-11-20T18:00:58,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=8579 2024-11-20T18:00:58,069 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:00:58,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-20T18:00:58,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:00:58,072 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:58,073 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-20T18:00:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T18:00:58,074 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:00:58,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741851_1027 (size=407) 2024-11-20T18:00:58,489 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => baa57eae0627d14c7383df10895823ba, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:00:58,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741852_1028 (size=58) 2024-11-20T18:00:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing baa57eae0627d14c7383df10895823ba, disabling compactions & flushes 2024-11-20T18:00:58,903 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. after waiting 0 ms 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:58,903 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:58,903 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for baa57eae0627d14c7383df10895823ba: Waiting for close lock at 1732125658903Disabling compacts and flushes for region at 1732125658903Disabling writes for close at 1732125658903Writing region close event to WAL at 1732125658903Closed at 1732125658903 2024-11-20T18:00:58,907 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:00:58,907 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732125658907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125658907"}]},"ts":"1732125658907"} 2024-11-20T18:00:58,911 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:00:58,912 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:00:58,912 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125658912"}]},"ts":"1732125658912"} 2024-11-20T18:00:58,915 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-20T18:00:58,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=baa57eae0627d14c7383df10895823ba, ASSIGN}] 2024-11-20T18:00:58,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=baa57eae0627d14c7383df10895823ba, ASSIGN 2024-11-20T18:00:58,917 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=baa57eae0627d14c7383df10895823ba, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:00:59,069 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=baa57eae0627d14c7383df10895823ba, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:59,074 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=baa57eae0627d14c7383df10895823ba, ASSIGN because future has completed 2024-11-20T18:00:59,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure baa57eae0627d14c7383df10895823ba, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:00:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T18:00:59,235 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:59,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => baa57eae0627d14c7383df10895823ba, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:00:59,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:00:59,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,237 INFO [StoreOpener-baa57eae0627d14c7383df10895823ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,239 INFO [StoreOpener-baa57eae0627d14c7383df10895823ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region baa57eae0627d14c7383df10895823ba columnFamilyName testFamily 2024-11-20T18:00:59,239 DEBUG [StoreOpener-baa57eae0627d14c7383df10895823ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:00:59,240 INFO [StoreOpener-baa57eae0627d14c7383df10895823ba-1 {}] regionserver.HStore(327): Store=baa57eae0627d14c7383df10895823ba/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:00:59,240 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,241 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,241 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,244 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:00:59,247 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened baa57eae0627d14c7383df10895823ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60607043, jitterRate=-0.09688468277454376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:00:59,247 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for baa57eae0627d14c7383df10895823ba 2024-11-20T18:00:59,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for baa57eae0627d14c7383df10895823ba: Running coprocessor pre-open hook at 1732125659236Writing region info on filesystem at 1732125659236Initializing all the Stores at 1732125659237 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125659237Cleaning up temporary data from old regions at 1732125659242 (+5 ms)Running coprocessor post-open hooks at 1732125659247 (+5 ms)Region opened successfully at 1732125659248 (+1 ms) 2024-11-20T18:00:59,249 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba., pid=30, masterSystemTime=1732125659229 2024-11-20T18:00:59,252 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:59,252 INFO [RS_OPEN_PRIORITY_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:00:59,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=baa57eae0627d14c7383df10895823ba, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:00:59,256 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure baa57eae0627d14c7383df10895823ba, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:00:59,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T18:00:59,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure baa57eae0627d14c7383df10895823ba, server=613581fcdc40,46615,1732125544057 in 183 msec 2024-11-20T18:00:59,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-20T18:00:59,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=baa57eae0627d14c7383df10895823ba, ASSIGN in 345 msec 2024-11-20T18:00:59,264 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:00:59,264 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125659264"}]},"ts":"1732125659264"} 2024-11-20T18:00:59,266 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-20T18:00:59,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:00:59,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1980 sec 2024-11-20T18:01:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-20T18:01:01,107 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-20T18:01:01,107 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-20T18:01:01,108 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:01:01,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-20T18:01:01,116 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:01:01,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-20T18:01:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:01:01,129 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:01:01,129 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-20T18:01:01,132 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-20T18:01:01,134 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-20T18:01:01,135 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-20T18:01:01,135 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-20T18:01:01,699 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-20T18:01:02,166 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T18:01:03,576 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-20T18:01:03,577 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-20T18:01:04,741 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:01:07,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417281996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-20T18:01:07,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46615 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:51634 deadline: 1732125673138, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417281996' does not exist 2024-11-20T18:01:07,139 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 613581fcdc40,46615,1732125544057 for closing scanner id = -8139664020417281996 for baa57eae0627d14c7383df10895823ba of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8139664020417281996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T18:01:07,143 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=8552 (was 8579) 2024-11-20T18:01:07,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=8551 2024-11-20T18:01:07,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T18:01:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-20T18:01:07,156 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T18:01:07,156 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:01:07,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-20T18:01:07,157 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T18:01:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T18:01:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741853_1029 (size=423) 2024-11-20T18:01:07,570 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab8cdb8f36efc3bffeff8ac1f4253ef9, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04 2024-11-20T18:01:07,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741854_1030 (size=74) 2024-11-20T18:01:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-20T18:01:07,983 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:01:07,983 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing ab8cdb8f36efc3bffeff8ac1f4253ef9, disabling compactions & flushes 2024-11-20T18:01:07,983 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:07,983 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:07,983 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. after waiting 0 ms 2024-11-20T18:01:07,983 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:07,983 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:07,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab8cdb8f36efc3bffeff8ac1f4253ef9: Waiting for close lock at 1732125667983Disabling compacts and flushes for region at 1732125667983Disabling writes for close at 1732125667983Writing region close event to WAL at 1732125667983Closed at 1732125667983 2024-11-20T18:01:07,985 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T18:01:07,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732125667985"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732125667985"}]},"ts":"1732125667985"} 2024-11-20T18:01:07,989 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T18:01:07,990 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T18:01:07,991 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125667991"}]},"ts":"1732125667991"} 2024-11-20T18:01:07,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-20T18:01:07,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=ab8cdb8f36efc3bffeff8ac1f4253ef9, ASSIGN}] 2024-11-20T18:01:07,996 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=ab8cdb8f36efc3bffeff8ac1f4253ef9, ASSIGN 2024-11-20T18:01:07,997 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=ab8cdb8f36efc3bffeff8ac1f4253ef9, ASSIGN; state=OFFLINE, location=613581fcdc40,46615,1732125544057; forceNewPlan=false, retain=false 2024-11-20T18:01:08,148 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=ab8cdb8f36efc3bffeff8ac1f4253ef9, regionState=OPENING, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:01:08,151 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=ab8cdb8f36efc3bffeff8ac1f4253ef9, ASSIGN because future has completed 2024-11-20T18:01:08,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab8cdb8f36efc3bffeff8ac1f4253ef9, server=613581fcdc40,46615,1732125544057}] 2024-11-20T18:01:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T18:01:08,309 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:08,309 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => ab8cdb8f36efc3bffeff8ac1f4253ef9, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9.', STARTKEY => '', ENDKEY => ''} 2024-11-20T18:01:08,310 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,310 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T18:01:08,310 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,310 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,312 INFO [StoreOpener-ab8cdb8f36efc3bffeff8ac1f4253ef9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,314 INFO [StoreOpener-ab8cdb8f36efc3bffeff8ac1f4253ef9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab8cdb8f36efc3bffeff8ac1f4253ef9 columnFamilyName testFamily 2024-11-20T18:01:08,314 DEBUG [StoreOpener-ab8cdb8f36efc3bffeff8ac1f4253ef9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T18:01:08,315 INFO [StoreOpener-ab8cdb8f36efc3bffeff8ac1f4253ef9-1 {}] regionserver.HStore(327): Store=ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T18:01:08,315 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,316 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,317 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,317 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,317 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,319 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,322 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T18:01:08,323 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened ab8cdb8f36efc3bffeff8ac1f4253ef9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65885653, jitterRate=-0.018227264285087585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T18:01:08,323 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:08,324 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for ab8cdb8f36efc3bffeff8ac1f4253ef9: Running coprocessor pre-open hook at 1732125668310Writing region info on filesystem at 1732125668310Initializing all the Stores at 1732125668312 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732125668312Cleaning up temporary data from old regions at 1732125668317 (+5 ms)Running coprocessor post-open hooks at 1732125668323 (+6 ms)Region opened successfully at 1732125668324 (+1 ms) 2024-11-20T18:01:08,325 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., pid=33, masterSystemTime=1732125668304 2024-11-20T18:01:08,328 DEBUG [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:08,328 INFO [RS_OPEN_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:08,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=ab8cdb8f36efc3bffeff8ac1f4253ef9, regionState=OPEN, openSeqNum=2, regionLocation=613581fcdc40,46615,1732125544057 2024-11-20T18:01:08,331 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab8cdb8f36efc3bffeff8ac1f4253ef9, server=613581fcdc40,46615,1732125544057 because future has completed 2024-11-20T18:01:08,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-20T18:01:08,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure ab8cdb8f36efc3bffeff8ac1f4253ef9, server=613581fcdc40,46615,1732125544057 in 181 msec 2024-11-20T18:01:08,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T18:01:08,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=ab8cdb8f36efc3bffeff8ac1f4253ef9, ASSIGN in 341 msec 2024-11-20T18:01:08,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T18:01:08,339 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732125668339"}]},"ts":"1732125668339"} 2024-11-20T18:01:08,341 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-20T18:01:08,343 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T18:01:08,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1900 sec 2024-11-20T18:01:09,362 DEBUG [master/613581fcdc40:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T18:01:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T18:01:10,187 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-20T18:01:10,187 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-20T18:01:10,188 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:01:10,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-20T18:01:10,198 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T18:01:10,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-20T18:01:10,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:01:10,214 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-20T18:01:10,214 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-20T18:01:10,218 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-20T18:01:10,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T18:01:11,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-20T18:01:11,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-20T18:01:11,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:01:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-20T18:01:12,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2] 2024-11-20T18:01:12,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46615 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-20T18:01:13,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=613581fcdc40:46615 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=980ms 2024-11-20T18:01:13,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=980ms 2024-11-20T18:01:13,226 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., hostname=613581fcdc40,46615,1732125544057, seqNum=2 from cache 2024-11-20T18:01:13,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 221) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 200) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8523 (was 8551) 2024-11-20T18:01:13,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T18:01:13,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T18:01:13,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T18:01:13,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T18:01:13,244 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T18:01:13,244 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T18:01:13,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T18:01:13,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,244 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T18:01:13,244 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-11-20T18:01:13,245 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=613581fcdc40,35971,1732125543153 2024-11-20T18:01:13,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T18:01:13,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T18:01:13,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T18:01:13,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T18:01:13,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T18:01:13,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T18:01:13,296 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T18:01:13,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T18:01:13,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T18:01:13,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,296 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '613581fcdc40,46615,1732125544057' ***** 2024-11-20T18:01:13,297 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T18:01:13,297 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T18:01:13,297 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T18:01:13,297 INFO [RS:0;613581fcdc40:46615 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T18:01:13,297 INFO [RS:0;613581fcdc40:46615 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T18:01:13,297 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for b87905cc81e3d78383b58552be822048 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for ef28d09bb8b7b63f2ec04d794276a93e 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for ab8cdb8f36efc3bffeff8ac1f4253ef9 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for fe119899672709659768ffe936fc77d3 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for c970d752579eca03ef286dd82dac0e9f 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for a7bacbf91cca191c9255ffd957a757d2 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for dcd532c37d83a3d6feb163c60d6720fa 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for 35c5b501588438fab9b324c90dbb0d72 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for baa57eae0627d14c7383df10895823ba 2024-11-20T18:01:13,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b87905cc81e3d78383b58552be822048, disabling compactions & flushes 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(959): stopping server 613581fcdc40,46615,1732125544057 2024-11-20T18:01:13,298 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T18:01:13,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:01:13,298 INFO [RS:0;613581fcdc40:46615 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;613581fcdc40:46615. 2024-11-20T18:01:13,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. after waiting 0 ms 2024-11-20T18:01:13,298 DEBUG [RS:0;613581fcdc40:46615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T18:01:13,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:01:13,298 DEBUG [RS:0;613581fcdc40:46615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:13,299 INFO [RS:0;613581fcdc40:46615 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T18:01:13,299 INFO [RS:0;613581fcdc40:46615 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T18:01:13,299 INFO [RS:0;613581fcdc40:46615 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T18:01:13,299 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T18:01:13,299 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-20T18:01:13,299 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T18:01:13,299 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T18:01:13,299 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, b87905cc81e3d78383b58552be822048, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, ef28d09bb8b7b63f2ec04d794276a93e, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:13,299 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T18:01:13,299 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T18:01:13,300 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T18:01:13,302 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b87905cc81e3d78383b58552be822048 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:13,302 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-20T18:01:13,360 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/.tmp/testFamily/efe2e258e73e41d1965e77aeb26e9f3a is 61, key is row-0/testFamily:testQualifier/1732125623576/Put/seqid=0 2024-11-20T18:01:13,368 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/info/d55b438c216a4b03a4fd5f20ef93f325 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa./info:regioninfo/1732125609503/Put/seqid=0 2024-11-20T18:01:13,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741855_1031 (size=5049) 2024-11-20T18:01:13,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741856_1032 (size=20564) 2024-11-20T18:01:13,500 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, b87905cc81e3d78383b58552be822048, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, ef28d09bb8b7b63f2ec04d794276a93e, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:13,700 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, b87905cc81e3d78383b58552be822048, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, ef28d09bb8b7b63f2ec04d794276a93e, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:13,739 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-20T18:01:13,770 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/.tmp/testFamily/efe2e258e73e41d1965e77aeb26e9f3a 2024-11-20T18:01:13,775 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/info/d55b438c216a4b03a4fd5f20ef93f325 2024-11-20T18:01:13,811 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:01:13,818 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/.tmp/testFamily/efe2e258e73e41d1965e77aeb26e9f3a as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/testFamily/efe2e258e73e41d1965e77aeb26e9f3a 2024-11-20T18:01:13,828 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/testFamily/efe2e258e73e41d1965e77aeb26e9f3a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:13,836 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b87905cc81e3d78383b58552be822048 in 533ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:13,841 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/ns/e9956607955a4e6f86137f0360570f97 is 43, key is default/ns:d/1732125607929/Put/seqid=0 2024-11-20T18:01:13,843 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNextAsync/b87905cc81e3d78383b58552be822048/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:13,849 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:01:13,849 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b87905cc81e3d78383b58552be822048: Waiting for close lock at 1732125673298Running coprocessor pre-close hooks at 1732125673298Disabling compacts and flushes for region at 1732125673298Disabling writes for close at 1732125673298Obtaining lock to block concurrent updates at 1732125673302 (+4 ms)Preparing flush snapshotting stores in b87905cc81e3d78383b58552be822048 at 1732125673302Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125673312 (+10 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. at 1732125673314 (+2 ms)Flushing b87905cc81e3d78383b58552be822048/testFamily: creating writer at 1732125673316 (+2 ms)Flushing b87905cc81e3d78383b58552be822048/testFamily: appending metadata at 1732125673354 (+38 ms)Flushing b87905cc81e3d78383b58552be822048/testFamily: closing flushed file at 1732125673356 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1235f136: reopening flushed file at 1732125673816 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b87905cc81e3d78383b58552be822048 in 533ms, sequenceid=8, compaction requested=false at 1732125673836 (+20 ms)Writing region close event to WAL at 1732125673838 (+2 ms)Running coprocessor post-close hooks at 1732125673844 (+6 ms)Closed at 1732125673849 (+5 ms) 2024-11-20T18:01:13,850 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732125620519.b87905cc81e3d78383b58552be822048. 2024-11-20T18:01:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741857_1033 (size=5153) 2024-11-20T18:01:13,850 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef28d09bb8b7b63f2ec04d794276a93e, disabling compactions & flushes 2024-11-20T18:01:13,850 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-20T18:01:13,850 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:01:13,850 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-20T18:01:13,850 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:01:13,850 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. after waiting 0 ms 2024-11-20T18:01:13,851 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:01:13,851 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ef28d09bb8b7b63f2ec04d794276a93e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:13,851 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-20T18:01:13,852 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-20T18:01:13,868 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/.tmp/testFamily/fc8853a5ba2d4d3b84bc61551cd7f1eb is 61, key is row-0/testFamily:testQualifier/1732125614467/Put/seqid=0 2024-11-20T18:01:13,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741858_1034 (size=5049) 2024-11-20T18:01:13,900 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, ef28d09bb8b7b63f2ec04d794276a93e, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:14,101 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, ef28d09bb8b7b63f2ec04d794276a93e, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:14,253 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/ns/e9956607955a4e6f86137f0360570f97 2024-11-20T18:01:14,273 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/.tmp/testFamily/fc8853a5ba2d4d3b84bc61551cd7f1eb 2024-11-20T18:01:14,283 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/.tmp/testFamily/fc8853a5ba2d4d3b84bc61551cd7f1eb as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/testFamily/fc8853a5ba2d4d3b84bc61551cd7f1eb 2024-11-20T18:01:14,286 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/table/23929742274c4ead9b33a17c9ad17946 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732125609524/Put/seqid=0 2024-11-20T18:01:14,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741859_1035 (size=6699) 2024-11-20T18:01:14,292 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/testFamily/fc8853a5ba2d4d3b84bc61551cd7f1eb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:14,293 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ef28d09bb8b7b63f2ec04d794276a93e in 442ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:14,297 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScanner/ef28d09bb8b7b63f2ec04d794276a93e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:14,298 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:01:14,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef28d09bb8b7b63f2ec04d794276a93e: Waiting for close lock at 1732125673850Running coprocessor pre-close hooks at 1732125673850Disabling compacts and flushes for region at 1732125673850Disabling writes for close at 1732125673851 (+1 ms)Obtaining lock to block concurrent updates at 1732125673851Preparing flush snapshotting stores in ef28d09bb8b7b63f2ec04d794276a93e at 1732125673851Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125673851Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. at 1732125673852 (+1 ms)Flushing ef28d09bb8b7b63f2ec04d794276a93e/testFamily: creating writer at 1732125673852Flushing ef28d09bb8b7b63f2ec04d794276a93e/testFamily: appending metadata at 1732125673867 (+15 ms)Flushing ef28d09bb8b7b63f2ec04d794276a93e/testFamily: closing flushed file at 1732125673867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46dac8d9: reopening flushed file at 1732125674281 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ef28d09bb8b7b63f2ec04d794276a93e in 442ms, sequenceid=8, compaction requested=false at 1732125674293 (+12 ms)Writing region close event to WAL at 1732125674293Running coprocessor post-close hooks at 1732125674298 (+5 ms)Closed at 1732125674298 2024-11-20T18:01:14,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732125611415.ef28d09bb8b7b63f2ec04d794276a93e. 2024-11-20T18:01:14,298 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab8cdb8f36efc3bffeff8ac1f4253ef9, disabling compactions & flushes 2024-11-20T18:01:14,298 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:14,299 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:14,299 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. after waiting 0 ms 2024-11-20T18:01:14,299 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:14,299 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ab8cdb8f36efc3bffeff8ac1f4253ef9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:14,301 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-20T18:01:14,301 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1325): Online Regions={ab8cdb8f36efc3bffeff8ac1f4253ef9=testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., fe119899672709659768ffe936fc77d3=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., 1588230740=hbase:meta,,1.1588230740, c970d752579eca03ef286dd82dac0e9f=testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f., a7bacbf91cca191c9255ffd957a757d2=testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2., dcd532c37d83a3d6feb163c60d6720fa=testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., 35c5b501588438fab9b324c90dbb0d72=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., e0d943f6ccd4d61aee4e842944711b2b=testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., baa57eae0627d14c7383df10895823ba=hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.} 2024-11-20T18:01:14,301 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:14,314 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/.tmp/testFamily/2416438403264c79ad9096ff662993ff is 61, key is row-0/testFamily:testQualifier/1732125670203/Put/seqid=0 2024-11-20T18:01:14,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741860_1036 (size=5049) 2024-11-20T18:01:14,350 INFO [regionserver/613581fcdc40:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T18:01:14,351 INFO [regionserver/613581fcdc40:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T18:01:14,502 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:14,694 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/table/23929742274c4ead9b33a17c9ad17946 2024-11-20T18:01:14,702 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, ab8cdb8f36efc3bffeff8ac1f4253ef9, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:14,710 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/info/d55b438c216a4b03a4fd5f20ef93f325 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/info/d55b438c216a4b03a4fd5f20ef93f325 2024-11-20T18:01:14,719 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/.tmp/testFamily/2416438403264c79ad9096ff662993ff 2024-11-20T18:01:14,719 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/info/d55b438c216a4b03a4fd5f20ef93f325, entries=100, sequenceid=56, filesize=20.1 K 2024-11-20T18:01:14,721 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/ns/e9956607955a4e6f86137f0360570f97 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/ns/e9956607955a4e6f86137f0360570f97 2024-11-20T18:01:14,727 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/.tmp/testFamily/2416438403264c79ad9096ff662993ff as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily/2416438403264c79ad9096ff662993ff 2024-11-20T18:01:14,729 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/ns/e9956607955a4e6f86137f0360570f97, entries=2, sequenceid=56, filesize=5.0 K 2024-11-20T18:01:14,730 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/.tmp/table/23929742274c4ead9b33a17c9ad17946 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/table/23929742274c4ead9b33a17c9ad17946 2024-11-20T18:01:14,737 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily/2416438403264c79ad9096ff662993ff, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:14,738 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/table/23929742274c4ead9b33a17c9ad17946, entries=20, sequenceid=56, filesize=6.5 K 2024-11-20T18:01:14,738 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab8cdb8f36efc3bffeff8ac1f4253ef9 in 439ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:14,739 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1439ms, sequenceid=56, compaction requested=false 2024-11-20T18:01:14,744 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScannerAsync/ab8cdb8f36efc3bffeff8ac1f4253ef9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:14,744 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-20T18:01:14,744 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab8cdb8f36efc3bffeff8ac1f4253ef9: Waiting for close lock at 1732125674298Running coprocessor pre-close hooks at 1732125674298Disabling compacts and flushes for region at 1732125674298Disabling writes for close at 1732125674299 (+1 ms)Obtaining lock to block concurrent updates at 1732125674299Preparing flush snapshotting stores in ab8cdb8f36efc3bffeff8ac1f4253ef9 at 1732125674299Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125674299Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. at 1732125674299Flushing ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily: creating writer at 1732125674300 (+1 ms)Flushing ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily: appending metadata at 1732125674313 (+13 ms)Flushing ab8cdb8f36efc3bffeff8ac1f4253ef9/testFamily: closing flushed file at 1732125674313Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3527c26: reopening flushed file at 1732125674726 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab8cdb8f36efc3bffeff8ac1f4253ef9 in 439ms, sequenceid=8, compaction requested=false at 1732125674738 (+12 ms)Writing region close event to WAL at 1732125674739 (+1 ms)Running coprocessor post-close hooks at 1732125674744 (+5 ms)Closed at 1732125674744 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732125667153.ab8cdb8f36efc3bffeff8ac1f4253ef9. 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T18:01:14,745 INFO [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe119899672709659768ffe936fc77d3, disabling compactions & flushes 2024-11-20T18:01:14,745 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732125673299Running coprocessor pre-close hooks at 1732125673299Disabling compacts and flushes for region at 1732125673299Disabling writes for close at 1732125673300 (+1 ms)Obtaining lock to block concurrent updates at 1732125673302 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732125673302Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732125673312 (+10 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732125673314 (+2 ms)Flushing 1588230740/info: creating writer at 1732125673316 (+2 ms)Flushing 1588230740/info: appending metadata at 1732125673360 (+44 ms)Flushing 1588230740/info: closing flushed file at 1732125673360Flushing 1588230740/ns: creating writer at 1732125673815 (+455 ms)Flushing 1588230740/ns: appending metadata at 1732125673839 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732125673839Flushing 1588230740/table: creating writer at 1732125674268 (+429 ms)Flushing 1588230740/table: appending metadata at 1732125674285 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732125674285Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5888d0ba: reopening flushed file at 1732125674709 (+424 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ff21d08: reopening flushed file at 1732125674720 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c5fa1f8: reopening flushed file at 1732125674729 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1439ms, sequenceid=56, compaction requested=false at 1732125674739 (+10 ms)Writing region close event to WAL at 1732125674740 (+1 ms)Running coprocessor post-close hooks at 1732125674745 (+5 ms)Closed at 1732125674745 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. after waiting 0 ms 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:01:14,745 DEBUG [RS_CLOSE_META-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T18:01:14,745 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fe119899672709659768ffe936fc77d3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:14,762 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/.tmp/testFamily/e3f6bff9ae8d4e4f94b0df3b3e159325 is 61, key is row-0/testFamily:testQualifier/1732125652028/Put/seqid=0 2024-11-20T18:01:14,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741861_1037 (size=5049) 2024-11-20T18:01:14,903 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:15,103 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b, fe119899672709659768ffe936fc77d3 2024-11-20T18:01:15,170 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/.tmp/testFamily/e3f6bff9ae8d4e4f94b0df3b3e159325 2024-11-20T18:01:15,183 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/.tmp/testFamily/e3f6bff9ae8d4e4f94b0df3b3e159325 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/testFamily/e3f6bff9ae8d4e4f94b0df3b3e159325 2024-11-20T18:01:15,193 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/testFamily/e3f6bff9ae8d4e4f94b0df3b3e159325, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:15,194 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe119899672709659768ffe936fc77d3 in 449ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:15,199 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/fe119899672709659768ffe936fc77d3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:15,200 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe119899672709659768ffe936fc77d3: Waiting for close lock at 1732125674745Running coprocessor pre-close hooks at 1732125674745Disabling compacts and flushes for region at 1732125674745Disabling writes for close at 1732125674745Obtaining lock to block concurrent updates at 1732125674745Preparing flush snapshotting stores in fe119899672709659768ffe936fc77d3 at 1732125674745Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125674746 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. at 1732125674746Flushing fe119899672709659768ffe936fc77d3/testFamily: creating writer at 1732125674746Flushing fe119899672709659768ffe936fc77d3/testFamily: appending metadata at 1732125674762 (+16 ms)Flushing fe119899672709659768ffe936fc77d3/testFamily: closing flushed file at 1732125674762Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f5ef2d: reopening flushed file at 1732125675182 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe119899672709659768ffe936fc77d3 in 449ms, sequenceid=8, compaction requested=false at 1732125675194 (+12 ms)Writing region close event to WAL at 1732125675195 (+1 ms)Running coprocessor post-close hooks at 1732125675200 (+5 ms)Closed at 1732125675200 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732125648976.fe119899672709659768ffe936fc77d3. 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c970d752579eca03ef286dd82dac0e9f, disabling compactions & flushes 2024-11-20T18:01:15,200 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. after waiting 0 ms 2024-11-20T18:01:15,200 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:01:15,200 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c970d752579eca03ef286dd82dac0e9f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:15,217 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/.tmp/testFamily/7914f2eb42974a179d7fcdb489e89a6a is 61, key is row-0/testFamily:testQualifier/1732125645930/Put/seqid=0 2024-11-20T18:01:15,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741862_1038 (size=5049) 2024-11-20T18:01:15,303 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-20T18:01:15,303 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1325): Online Regions={c970d752579eca03ef286dd82dac0e9f=testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f., a7bacbf91cca191c9255ffd957a757d2=testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2., dcd532c37d83a3d6feb163c60d6720fa=testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., 35c5b501588438fab9b324c90dbb0d72=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., e0d943f6ccd4d61aee4e842944711b2b=testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., baa57eae0627d14c7383df10895823ba=hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.} 2024-11-20T18:01:15,304 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:15,504 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, c970d752579eca03ef286dd82dac0e9f, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:15,624 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/.tmp/testFamily/7914f2eb42974a179d7fcdb489e89a6a 2024-11-20T18:01:15,640 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/.tmp/testFamily/7914f2eb42974a179d7fcdb489e89a6a as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/testFamily/7914f2eb42974a179d7fcdb489e89a6a 2024-11-20T18:01:15,649 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/testFamily/7914f2eb42974a179d7fcdb489e89a6a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:15,650 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c970d752579eca03ef286dd82dac0e9f in 450ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:15,655 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNext/c970d752579eca03ef286dd82dac0e9f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:15,655 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c970d752579eca03ef286dd82dac0e9f: Waiting for close lock at 1732125675200Running coprocessor pre-close hooks at 1732125675200Disabling compacts and flushes for region at 1732125675200Disabling writes for close at 1732125675200Obtaining lock to block concurrent updates at 1732125675200Preparing flush snapshotting stores in c970d752579eca03ef286dd82dac0e9f at 1732125675200Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125675201 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. at 1732125675201Flushing c970d752579eca03ef286dd82dac0e9f/testFamily: creating writer at 1732125675201Flushing c970d752579eca03ef286dd82dac0e9f/testFamily: appending metadata at 1732125675217 (+16 ms)Flushing c970d752579eca03ef286dd82dac0e9f/testFamily: closing flushed file at 1732125675217Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ee904a7: reopening flushed file at 1732125675639 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c970d752579eca03ef286dd82dac0e9f in 450ms, sequenceid=8, compaction requested=false at 1732125675650 (+11 ms)Writing region close event to WAL at 1732125675651 (+1 ms)Running coprocessor post-close hooks at 1732125675655 (+4 ms)Closed at 1732125675655 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732125642892.c970d752579eca03ef286dd82dac0e9f. 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a7bacbf91cca191c9255ffd957a757d2, disabling compactions & flushes 2024-11-20T18:01:15,656 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. after waiting 0 ms 2024-11-20T18:01:15,656 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:01:15,657 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a7bacbf91cca191c9255ffd957a757d2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:15,672 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/.tmp/testFamily/aa11ce03b8cb45969928260dfc5ee404 is 61, key is row-0/testFamily:testQualifier/1732125632672/Put/seqid=0 2024-11-20T18:01:15,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741863_1039 (size=5049) 2024-11-20T18:01:15,704 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:15,905 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,079 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/.tmp/testFamily/aa11ce03b8cb45969928260dfc5ee404 2024-11-20T18:01:16,096 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/.tmp/testFamily/aa11ce03b8cb45969928260dfc5ee404 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/testFamily/aa11ce03b8cb45969928260dfc5ee404 2024-11-20T18:01:16,104 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/testFamily/aa11ce03b8cb45969928260dfc5ee404, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:16,105 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a7bacbf91cca191c9255ffd957a757d2 in 449ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:16,105 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, a7bacbf91cca191c9255ffd957a757d2, baa57eae0627d14c7383df10895823ba, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,110 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextException/a7bacbf91cca191c9255ffd957a757d2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:16,110 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a7bacbf91cca191c9255ffd957a757d2: Waiting for close lock at 1732125675656Running coprocessor pre-close hooks at 1732125675656Disabling compacts and flushes for region at 1732125675656Disabling writes for close at 1732125675656Obtaining lock to block concurrent updates at 1732125675657 (+1 ms)Preparing flush snapshotting stores in a7bacbf91cca191c9255ffd957a757d2 at 1732125675657Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125675657Flushing stores of testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. at 1732125675658 (+1 ms)Flushing a7bacbf91cca191c9255ffd957a757d2/testFamily: creating writer at 1732125675658Flushing a7bacbf91cca191c9255ffd957a757d2/testFamily: appending metadata at 1732125675672 (+14 ms)Flushing a7bacbf91cca191c9255ffd957a757d2/testFamily: closing flushed file at 1732125675672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dd572c1: reopening flushed file at 1732125676094 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a7bacbf91cca191c9255ffd957a757d2 in 449ms, sequenceid=8, compaction requested=false at 1732125676105 (+11 ms)Writing region close event to WAL at 1732125676106 (+1 ms)Running coprocessor post-close hooks at 1732125676110 (+4 ms)Closed at 1732125676110 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732125629623.a7bacbf91cca191c9255ffd957a757d2. 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcd532c37d83a3d6feb163c60d6720fa, disabling compactions & flushes 2024-11-20T18:01:16,111 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. after waiting 0 ms 2024-11-20T18:01:16,111 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:01:16,111 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dcd532c37d83a3d6feb163c60d6720fa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:16,126 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/.tmp/testFamily/072a4b4826fd42fa93e269e6e44597bc is 61, key is row-0/testFamily:testQualifier/1732125611340/Put/seqid=0 2024-11-20T18:01:16,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741864_1040 (size=5049) 2024-11-20T18:01:16,306 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-20T18:01:16,306 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1325): Online Regions={dcd532c37d83a3d6feb163c60d6720fa=testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., 35c5b501588438fab9b324c90dbb0d72=testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., e0d943f6ccd4d61aee4e842944711b2b=testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., baa57eae0627d14c7383df10895823ba=hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.} 2024-11-20T18:01:16,306 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, baa57eae0627d14c7383df10895823ba, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,507 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, baa57eae0627d14c7383df10895823ba, dcd532c37d83a3d6feb163c60d6720fa, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,532 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/.tmp/testFamily/072a4b4826fd42fa93e269e6e44597bc 2024-11-20T18:01:16,542 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/.tmp/testFamily/072a4b4826fd42fa93e269e6e44597bc as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/testFamily/072a4b4826fd42fa93e269e6e44597bc 2024-11-20T18:01:16,551 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/testFamily/072a4b4826fd42fa93e269e6e44597bc, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:16,553 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dcd532c37d83a3d6feb163c60d6720fa in 441ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:16,558 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testRetryOutOfOrderScannerNextExceptionAsync/dcd532c37d83a3d6feb163c60d6720fa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:16,559 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:01:16,559 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcd532c37d83a3d6feb163c60d6720fa: Waiting for close lock at 1732125676111Running coprocessor pre-close hooks at 1732125676111Disabling compacts and flushes for region at 1732125676111Disabling writes for close at 1732125676111Obtaining lock to block concurrent updates at 1732125676111Preparing flush snapshotting stores in dcd532c37d83a3d6feb163c60d6720fa at 1732125676111Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125676111Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. at 1732125676112 (+1 ms)Flushing dcd532c37d83a3d6feb163c60d6720fa/testFamily: creating writer at 1732125676112Flushing dcd532c37d83a3d6feb163c60d6720fa/testFamily: appending metadata at 1732125676126 (+14 ms)Flushing dcd532c37d83a3d6feb163c60d6720fa/testFamily: closing flushed file at 1732125676126Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@112b9c3d: reopening flushed file at 1732125676540 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dcd532c37d83a3d6feb163c60d6720fa in 441ms, sequenceid=8, compaction requested=false at 1732125676553 (+13 ms)Writing region close event to WAL at 1732125676553Running coprocessor post-close hooks at 1732125676559 (+6 ms)Closed at 1732125676559 2024-11-20T18:01:16,559 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732125608260.dcd532c37d83a3d6feb163c60d6720fa. 2024-11-20T18:01:16,559 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 35c5b501588438fab9b324c90dbb0d72, disabling compactions & flushes 2024-11-20T18:01:16,559 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:01:16,559 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:01:16,559 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. after waiting 0 ms 2024-11-20T18:01:16,560 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:01:16,560 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 35c5b501588438fab9b324c90dbb0d72 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:16,577 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/.tmp/testFamily/0c022730f884442ab501b8f24f228bf6 is 61, key is row-0/testFamily:testQualifier/1732125635781/Put/seqid=0 2024-11-20T18:01:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741865_1041 (size=5049) 2024-11-20T18:01:16,707 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, baa57eae0627d14c7383df10895823ba, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,907 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on 35c5b501588438fab9b324c90dbb0d72, baa57eae0627d14c7383df10895823ba, e0d943f6ccd4d61aee4e842944711b2b 2024-11-20T18:01:16,984 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/.tmp/testFamily/0c022730f884442ab501b8f24f228bf6 2024-11-20T18:01:16,998 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/.tmp/testFamily/0c022730f884442ab501b8f24f228bf6 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/testFamily/0c022730f884442ab501b8f24f228bf6 2024-11-20T18:01:17,006 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/testFamily/0c022730f884442ab501b8f24f228bf6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:17,008 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 35c5b501588438fab9b324c90dbb0d72 in 448ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:17,014 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnOpenScanner/35c5b501588438fab9b324c90dbb0d72/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:17,015 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 35c5b501588438fab9b324c90dbb0d72: Waiting for close lock at 1732125676559Running coprocessor pre-close hooks at 1732125676559Disabling compacts and flushes for region at 1732125676559Disabling writes for close at 1732125676560 (+1 ms)Obtaining lock to block concurrent updates at 1732125676560Preparing flush snapshotting stores in 35c5b501588438fab9b324c90dbb0d72 at 1732125676560Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125676560Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. at 1732125676561 (+1 ms)Flushing 35c5b501588438fab9b324c90dbb0d72/testFamily: creating writer at 1732125676561Flushing 35c5b501588438fab9b324c90dbb0d72/testFamily: appending metadata at 1732125676577 (+16 ms)Flushing 35c5b501588438fab9b324c90dbb0d72/testFamily: closing flushed file at 1732125676577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23586ab3: reopening flushed file at 1732125676996 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 35c5b501588438fab9b324c90dbb0d72 in 448ms, sequenceid=8, compaction requested=false at 1732125677008 (+12 ms)Writing region close event to WAL at 1732125677009 (+1 ms)Running coprocessor post-close hooks at 1732125677015 (+6 ms)Closed at 1732125677015 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732125632729.35c5b501588438fab9b324c90dbb0d72. 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0d943f6ccd4d61aee4e842944711b2b, disabling compactions & flushes 2024-11-20T18:01:17,016 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. after waiting 0 ms 2024-11-20T18:01:17,016 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:01:17,016 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e0d943f6ccd4d61aee4e842944711b2b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:17,040 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/.tmp/testFamily/6950c2ae06034589983b0ea8f87eb374 is 61, key is row-0/testFamily:testQualifier/1732125639852/Put/seqid=0 2024-11-20T18:01:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741866_1042 (size=5049) 2024-11-20T18:01:17,047 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/.tmp/testFamily/6950c2ae06034589983b0ea8f87eb374 2024-11-20T18:01:17,057 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/.tmp/testFamily/6950c2ae06034589983b0ea8f87eb374 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/testFamily/6950c2ae06034589983b0ea8f87eb374 2024-11-20T18:01:17,065 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/testFamily/6950c2ae06034589983b0ea8f87eb374, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:17,066 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e0d943f6ccd4d61aee4e842944711b2b in 50ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:17,071 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/default/testNormalScanTimeoutOnNextAsync/e0d943f6ccd4d61aee4e842944711b2b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:17,072 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:01:17,072 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0d943f6ccd4d61aee4e842944711b2b: Waiting for close lock at 1732125677016Running coprocessor pre-close hooks at 1732125677016Disabling compacts and flushes for region at 1732125677016Disabling writes for close at 1732125677016Obtaining lock to block concurrent updates at 1732125677016Preparing flush snapshotting stores in e0d943f6ccd4d61aee4e842944711b2b at 1732125677016Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125677017 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. at 1732125677017Flushing e0d943f6ccd4d61aee4e842944711b2b/testFamily: creating writer at 1732125677018 (+1 ms)Flushing e0d943f6ccd4d61aee4e842944711b2b/testFamily: appending metadata at 1732125677040 (+22 ms)Flushing e0d943f6ccd4d61aee4e842944711b2b/testFamily: closing flushed file at 1732125677040Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36cc76b7: reopening flushed file at 1732125677055 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e0d943f6ccd4d61aee4e842944711b2b in 50ms, sequenceid=8, compaction requested=false at 1732125677066 (+11 ms)Writing region close event to WAL at 1732125677067 (+1 ms)Running coprocessor post-close hooks at 1732125677072 (+5 ms)Closed at 1732125677072 2024-11-20T18:01:17,072 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732125638819.e0d943f6ccd4d61aee4e842944711b2b. 2024-11-20T18:01:17,073 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing baa57eae0627d14c7383df10895823ba, disabling compactions & flushes 2024-11-20T18:01:17,073 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:01:17,073 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:01:17,073 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. after waiting 0 ms 2024-11-20T18:01:17,073 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:01:17,073 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing baa57eae0627d14c7383df10895823ba 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-20T18:01:17,094 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/.tmp/testFamily/c2b1bf227079421baf4f8c1a350abe71 is 61, key is row-0/testFamily:testQualifier/1732125661121/Put/seqid=0 2024-11-20T18:01:17,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741867_1043 (size=5049) 2024-11-20T18:01:17,108 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on baa57eae0627d14c7383df10895823ba 2024-11-20T18:01:17,308 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T18:01:17,308 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1325): Online Regions={baa57eae0627d14c7383df10895823ba=hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba.} 2024-11-20T18:01:17,308 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on baa57eae0627d14c7383df10895823ba 2024-11-20T18:01:17,502 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/.tmp/testFamily/c2b1bf227079421baf4f8c1a350abe71 2024-11-20T18:01:17,509 DEBUG [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1351): Waiting on baa57eae0627d14c7383df10895823ba 2024-11-20T18:01:17,518 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/.tmp/testFamily/c2b1bf227079421baf4f8c1a350abe71 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/testFamily/c2b1bf227079421baf4f8c1a350abe71 2024-11-20T18:01:17,527 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/testFamily/c2b1bf227079421baf4f8c1a350abe71, entries=4, sequenceid=8, filesize=4.9 K 2024-11-20T18:01:17,529 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for baa57eae0627d14c7383df10895823ba in 456ms, sequenceid=8, compaction requested=false 2024-11-20T18:01:17,534 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/data/hbase/testMetaScanTimeoutOnNext/baa57eae0627d14c7383df10895823ba/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-20T18:01:17,535 INFO [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:01:17,535 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for baa57eae0627d14c7383df10895823ba: Waiting for close lock at 1732125677073Running coprocessor pre-close hooks at 1732125677073Disabling compacts and flushes for region at 1732125677073Disabling writes for close at 1732125677073Obtaining lock to block concurrent updates at 1732125677073Preparing flush snapshotting stores in baa57eae0627d14c7383df10895823ba at 1732125677073Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732125677073Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. at 1732125677074 (+1 ms)Flushing baa57eae0627d14c7383df10895823ba/testFamily: creating writer at 1732125677074Flushing baa57eae0627d14c7383df10895823ba/testFamily: appending metadata at 1732125677094 (+20 ms)Flushing baa57eae0627d14c7383df10895823ba/testFamily: closing flushed file at 1732125677094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f8e8b42: reopening flushed file at 1732125677516 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for baa57eae0627d14c7383df10895823ba in 456ms, sequenceid=8, compaction requested=false at 1732125677529 (+13 ms)Writing region close event to WAL at 1732125677530 (+1 ms)Running coprocessor post-close hooks at 1732125677535 (+5 ms)Closed at 1732125677535 2024-11-20T18:01:17,535 DEBUG [RS_CLOSE_REGION-regionserver/613581fcdc40:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732125658069.baa57eae0627d14c7383df10895823ba. 2024-11-20T18:01:17,709 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(976): stopping server 613581fcdc40,46615,1732125544057; all regions closed. 2024-11-20T18:01:17,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741834_1010 (size=19479) 2024-11-20T18:01:17,728 DEBUG [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/oldWALs 2024-11-20T18:01:17,728 INFO [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C46615%2C1732125544057.meta:.meta(num 1732125607691) 2024-11-20T18:01:17,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741833_1009 (size=20031) 2024-11-20T18:01:17,734 DEBUG [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/oldWALs 2024-11-20T18:01:17,734 INFO [RS:0;613581fcdc40:46615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 613581fcdc40%2C46615%2C1732125544057:(num 1732125607326) 2024-11-20T18:01:17,734 DEBUG [RS:0;613581fcdc40:46615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T18:01:17,734 INFO [RS:0;613581fcdc40:46615 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T18:01:17,734 INFO [RS:0;613581fcdc40:46615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T18:01:17,735 INFO [RS:0;613581fcdc40:46615 {}] hbase.ChoreService(370): Chore service for: regionserver/613581fcdc40:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T18:01:17,735 INFO [RS:0;613581fcdc40:46615 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T18:01:17,735 INFO [regionserver/613581fcdc40:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T18:01:17,736 INFO [RS:0;613581fcdc40:46615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46615 2024-11-20T18:01:17,783 INFO [regionserver/613581fcdc40:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T18:01:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/613581fcdc40,46615,1732125544057 2024-11-20T18:01:17,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T18:01:17,790 INFO [RS:0;613581fcdc40:46615 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T18:01:17,791 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [613581fcdc40,46615,1732125544057] 2024-11-20T18:01:17,813 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/613581fcdc40,46615,1732125544057 already deleted, retry=false 2024-11-20T18:01:17,814 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 613581fcdc40,46615,1732125544057 expired; onlineServers=0 2024-11-20T18:01:17,814 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '613581fcdc40,35971,1732125543153' ***** 2024-11-20T18:01:17,814 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T18:01:17,815 INFO [M:0;613581fcdc40:35971 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T18:01:17,815 INFO [M:0;613581fcdc40:35971 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T18:01:17,815 DEBUG [M:0;613581fcdc40:35971 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T18:01:17,815 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T18:01:17,816 DEBUG [M:0;613581fcdc40:35971 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T18:01:17,816 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732125547090 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.large.0-1732125547090,5,FailOnTimeoutGroup] 2024-11-20T18:01:17,816 DEBUG [master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732125547101 {}] cleaner.HFileCleaner(306): Exit Thread[master/613581fcdc40:0:becomeActiveMaster-HFileCleaner.small.0-1732125547101,5,FailOnTimeoutGroup] 2024-11-20T18:01:17,816 INFO [M:0;613581fcdc40:35971 {}] hbase.ChoreService(370): Chore service for: master/613581fcdc40:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T18:01:17,816 INFO [M:0;613581fcdc40:35971 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T18:01:17,817 DEBUG [M:0;613581fcdc40:35971 {}] master.HMaster(1795): Stopping service threads 2024-11-20T18:01:17,817 INFO [M:0;613581fcdc40:35971 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T18:01:17,817 INFO [M:0;613581fcdc40:35971 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T18:01:17,819 INFO [M:0;613581fcdc40:35971 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T18:01:17,819 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T18:01:17,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T18:01:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T18:01:17,824 DEBUG [M:0;613581fcdc40:35971 {}] zookeeper.ZKUtil(347): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T18:01:17,824 WARN [M:0;613581fcdc40:35971 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T18:01:17,825 INFO [M:0;613581fcdc40:35971 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/.lastflushedseqids 2024-11-20T18:01:17,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741868_1044 (size=649) 2024-11-20T18:01:17,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T18:01:17,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46615-0x1015966431a0001, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T18:01:17,904 INFO [RS:0;613581fcdc40:46615 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T18:01:17,905 INFO [RS:0;613581fcdc40:46615 {}] regionserver.HRegionServer(1031): Exiting; stopping=613581fcdc40,46615,1732125544057; zookeeper connection closed. 2024-11-20T18:01:17,905 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c0be05a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c0be05a 2024-11-20T18:01:17,906 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T18:01:18,235 INFO [M:0;613581fcdc40:35971 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T18:01:18,235 INFO [M:0;613581fcdc40:35971 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T18:01:18,236 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T18:01:18,236 INFO [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T18:01:18,236 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T18:01:18,236 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T18:01:18,236 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T18:01:18,236 INFO [M:0;613581fcdc40:35971 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.01 KB heapSize=190.91 KB 2024-11-20T18:01:18,253 DEBUG [M:0;613581fcdc40:35971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e3aa034d1144abc880675aa5bdb28d7 is 82, key is hbase:meta,,1/info:regioninfo/1732125607788/Put/seqid=0 2024-11-20T18:01:18,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741869_1045 (size=5672) 2024-11-20T18:01:18,658 INFO [M:0;613581fcdc40:35971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e3aa034d1144abc880675aa5bdb28d7 2024-11-20T18:01:18,686 DEBUG [M:0;613581fcdc40:35971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/268ab6c966414e7eacacc1a038918a34 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732125609533/Put/seqid=0 2024-11-20T18:01:18,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741870_1046 (size=14072) 2024-11-20T18:01:19,093 INFO [M:0;613581fcdc40:35971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.46 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/268ab6c966414e7eacacc1a038918a34 2024-11-20T18:01:19,106 INFO [M:0;613581fcdc40:35971 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 268ab6c966414e7eacacc1a038918a34 2024-11-20T18:01:19,123 DEBUG [M:0;613581fcdc40:35971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/96c1ffb5525541c597b37cccad138f9b is 69, key is 613581fcdc40,46615,1732125544057/rs:state/1732125577133/Put/seqid=0 2024-11-20T18:01:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741871_1047 (size=5156) 2024-11-20T18:01:19,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T18:01:19,529 INFO [M:0;613581fcdc40:35971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/96c1ffb5525541c597b37cccad138f9b 2024-11-20T18:01:19,537 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6e3aa034d1144abc880675aa5bdb28d7 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e3aa034d1144abc880675aa5bdb28d7 2024-11-20T18:01:19,544 INFO [M:0;613581fcdc40:35971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6e3aa034d1144abc880675aa5bdb28d7, entries=8, sequenceid=317, filesize=5.5 K 2024-11-20T18:01:19,546 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/268ab6c966414e7eacacc1a038918a34 as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/268ab6c966414e7eacacc1a038918a34 2024-11-20T18:01:19,553 INFO [M:0;613581fcdc40:35971 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 268ab6c966414e7eacacc1a038918a34 2024-11-20T18:01:19,553 INFO [M:0;613581fcdc40:35971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/268ab6c966414e7eacacc1a038918a34, entries=33, sequenceid=317, filesize=13.7 K 2024-11-20T18:01:19,554 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/96c1ffb5525541c597b37cccad138f9b as hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/96c1ffb5525541c597b37cccad138f9b 2024-11-20T18:01:19,561 INFO [M:0;613581fcdc40:35971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37129/user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/96c1ffb5525541c597b37cccad138f9b, entries=1, sequenceid=317, filesize=5.0 K 2024-11-20T18:01:19,562 INFO [M:0;613581fcdc40:35971 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.01 KB/166927, heapSize ~190.61 KB/195184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1326ms, sequenceid=317, compaction requested=false 2024-11-20T18:01:19,564 INFO [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T18:01:19,564 DEBUG [M:0;613581fcdc40:35971 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732125678236Disabling compacts and flushes for region at 1732125678236Disabling writes for close at 1732125678236Obtaining lock to block concurrent updates at 1732125678236Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732125678236Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166927, getHeapSize=195424, getOffHeapSize=0, getCellsCount=364 at 1732125678236Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732125678237 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732125678237Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732125678252 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732125678252Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732125678665 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732125678685 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732125678685Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732125679106 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732125679122 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732125679122Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df2f253: reopening flushed file at 1732125679536 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e2ca511: reopening flushed file at 1732125679544 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ce699dc: reopening flushed file at 1732125679553 (+9 ms)Finished flush of dataSize ~163.01 KB/166927, heapSize ~190.61 KB/195184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1326ms, sequenceid=317, compaction requested=false at 1732125679562 (+9 ms)Writing region close event to WAL at 1732125679564 (+2 ms)Closed at 1732125679564 2024-11-20T18:01:19,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34453 is added to blk_1073741830_1006 (size=188139) 2024-11-20T18:01:19,567 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/8db46dae-7b76-3e33-be7c-96d399d86a04/MasterData/WALs/613581fcdc40,35971,1732125543153/613581fcdc40%2C35971%2C1732125543153.1732125546257 not finished, retry = 0 2024-11-20T18:01:19,669 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T18:01:19,669 INFO [M:0;613581fcdc40:35971 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T18:01:19,669 INFO [M:0;613581fcdc40:35971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35971 2024-11-20T18:01:19,669 INFO [M:0;613581fcdc40:35971 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T18:01:19,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T18:01:19,879 INFO [M:0;613581fcdc40:35971 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T18:01:19,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35971-0x1015966431a0000, quorum=127.0.0.1:51016, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T18:01:19,887 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5359669a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T18:01:19,891 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a8bacf4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T18:01:19,892 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T18:01:19,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c25baf6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T18:01:19,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6506e00{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.log.dir/,STOPPED} 2024-11-20T18:01:19,896 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T18:01:19,896 WARN [BP-1234759647-172.17.0.2-1732125539043 heartbeating to localhost/127.0.0.1:37129 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T18:01:19,896 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T18:01:19,896 WARN [BP-1234759647-172.17.0.2-1732125539043 heartbeating to localhost/127.0.0.1:37129 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1234759647-172.17.0.2-1732125539043 (Datanode Uuid 22084a06-b039-4d4a-9b3b-52237c9bf74b) service to localhost/127.0.0.1:37129 2024-11-20T18:01:19,897 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/data/data1/current/BP-1234759647-172.17.0.2-1732125539043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T18:01:19,897 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/cluster_d21230d8-e1c5-c5f2-3603-46908a5773ce/data/data2/current/BP-1234759647-172.17.0.2-1732125539043 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T18:01:19,898 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T18:01:19,908 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T18:01:19,909 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T18:01:19,909 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T18:01:19,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T18:01:19,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ad43363c-761f-b577-afc9-44db388fd18b/hadoop.log.dir/,STOPPED} 2024-11-20T18:01:19,916 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T18:01:19,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down