2024-11-09 15:38:45,732 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-09 15:38:45,748 main DEBUG Took 0.013909 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 15:38:45,748 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 15:38:45,749 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 15:38:45,750 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 15:38:45,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,765 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 15:38:45,778 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,780 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,782 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,783 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,784 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,786 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,789 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,790 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,791 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,792 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,792 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,793 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 15:38:45,794 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,795 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 15:38:45,797 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 15:38:45,799 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 15:38:45,801 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 15:38:45,802 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 15:38:45,803 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 15:38:45,804 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 15:38:45,816 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 15:38:45,819 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 15:38:45,821 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 15:38:45,821 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 15:38:45,822 main DEBUG createAppenders(={Console}) 2024-11-09 15:38:45,823 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-09 15:38:45,823 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-09 15:38:45,824 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-09 15:38:45,824 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 15:38:45,825 main DEBUG OutputStream closed 2024-11-09 15:38:45,825 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 15:38:45,825 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 15:38:45,826 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-09 15:38:45,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 15:38:45,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 15:38:45,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 15:38:45,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 15:38:45,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 15:38:45,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 15:38:45,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 15:38:45,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 15:38:45,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 15:38:45,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 15:38:45,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 15:38:45,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 15:38:45,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 15:38:45,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 15:38:45,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 15:38:45,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 15:38:45,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 15:38:45,912 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 15:38:45,915 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 15:38:45,915 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-09 15:38:45,915 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 15:38:45,916 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-09T15:38:45,930 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-09 15:38:45,933 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 15:38:45,933 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T15:38:46,236 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c 2024-11-09T15:38:46,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T15:38:46,268 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2, deleteOnExit=true 2024-11-09T15:38:46,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T15:38:46,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/test.cache.data in system properties and HBase conf 2024-11-09T15:38:46,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T15:38:46,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.log.dir in system properties and HBase conf 2024-11-09T15:38:46,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T15:38:46,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T15:38:46,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T15:38:46,359 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T15:38:46,445 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T15:38:46,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T15:38:46,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T15:38:46,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T15:38:46,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T15:38:46,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T15:38:46,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T15:38:46,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T15:38:46,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T15:38:46,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T15:38:46,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/nfs.dump.dir in system properties and HBase conf 2024-11-09T15:38:46,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/java.io.tmpdir in system properties and HBase conf 2024-11-09T15:38:46,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T15:38:46,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T15:38:46,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T15:38:47,291 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T15:38:47,368 INFO [Time-limited test {}] log.Log(170): Logging initialized @2352ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T15:38:47,445 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T15:38:47,507 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T15:38:47,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T15:38:47,531 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T15:38:47,533 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T15:38:47,546 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T15:38:47,549 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.log.dir/,AVAILABLE} 2024-11-09T15:38:47,550 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T15:38:47,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/java.io.tmpdir/jetty-localhost-39165-hadoop-hdfs-3_4_1-tests_jar-_-any-18324477043546879507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T15:38:47,766 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:39165} 2024-11-09T15:38:47,766 INFO [Time-limited test {}] server.Server(415): Started @2752ms 2024-11-09T15:38:48,156 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T15:38:48,163 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T15:38:48,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T15:38:48,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T15:38:48,165 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T15:38:48,166 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.log.dir/,AVAILABLE} 2024-11-09T15:38:48,167 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T15:38:48,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/java.io.tmpdir/jetty-localhost-34305-hadoop-hdfs-3_4_1-tests_jar-_-any-9346390493210173053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T15:38:48,288 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:34305} 2024-11-09T15:38:48,288 INFO [Time-limited test {}] server.Server(415): Started @3274ms 2024-11-09T15:38:48,347 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T15:38:48,805 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/data/data2/current/BP-702925388-172.17.0.2-1731166727042/current, will proceed with Du for space computation calculation, 2024-11-09T15:38:48,805 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/data/data1/current/BP-702925388-172.17.0.2-1731166727042/current, will proceed with Du for space computation calculation, 2024-11-09T15:38:48,845 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T15:38:48,911 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfcc6686e00e05815 with lease ID 0x85c6c030159494a0: Processing first storage report for DS-936698ce-4a19-44b2-8dfc-1f99bfbd77cf from datanode DatanodeRegistration(127.0.0.1:39823, datanodeUuid=c9adfbac-fbae-4e2c-8e76-c79a7f953271, infoPort=43781, infoSecurePort=0, ipcPort=37789, storageInfo=lv=-57;cid=testClusterID;nsid=911200944;c=1731166727042) 2024-11-09T15:38:48,912 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfcc6686e00e05815 with lease ID 0x85c6c030159494a0: from storage DS-936698ce-4a19-44b2-8dfc-1f99bfbd77cf node DatanodeRegistration(127.0.0.1:39823, datanodeUuid=c9adfbac-fbae-4e2c-8e76-c79a7f953271, infoPort=43781, infoSecurePort=0, ipcPort=37789, storageInfo=lv=-57;cid=testClusterID;nsid=911200944;c=1731166727042), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-09T15:38:48,913 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfcc6686e00e05815 with lease ID 0x85c6c030159494a0: Processing first storage report for DS-76a18578-366e-4062-94d6-618670715233 from datanode DatanodeRegistration(127.0.0.1:39823, datanodeUuid=c9adfbac-fbae-4e2c-8e76-c79a7f953271, infoPort=43781, infoSecurePort=0, ipcPort=37789, storageInfo=lv=-57;cid=testClusterID;nsid=911200944;c=1731166727042) 2024-11-09T15:38:48,913 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfcc6686e00e05815 with lease ID 0x85c6c030159494a0: from storage DS-76a18578-366e-4062-94d6-618670715233 node DatanodeRegistration(127.0.0.1:39823, datanodeUuid=c9adfbac-fbae-4e2c-8e76-c79a7f953271, infoPort=43781, infoSecurePort=0, ipcPort=37789, storageInfo=lv=-57;cid=testClusterID;nsid=911200944;c=1731166727042), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T15:38:48,932 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c 2024-11-09T15:38:49,010 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/zookeeper_0, clientPort=53003, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T15:38:49,021 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53003 2024-11-09T15:38:49,036 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:49,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:49,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741825_1001 (size=7) 2024-11-09T15:38:49,677 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a with version=8 2024-11-09T15:38:49,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/hbase-staging 2024-11-09T15:38:49,785 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T15:38:50,036 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da6b20b51b98:0 server-side Connection retries=9 2024-11-09T15:38:50,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,052 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T15:38:50,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T15:38:50,190 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T15:38:50,251 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T15:38:50,260 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T15:38:50,264 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T15:38:50,291 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12355 (auto-detected) 2024-11-09T15:38:50,293 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T15:38:50,312 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34975 2024-11-09T15:38:50,334 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34975 connecting to ZooKeeper ensemble=127.0.0.1:53003 2024-11-09T15:38:50,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349750x0, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T15:38:50,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34975-0x1018daf68880000 connected 2024-11-09T15:38:50,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:50,404 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:50,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T15:38:50,417 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a, hbase.cluster.distributed=false 2024-11-09T15:38:50,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T15:38:50,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-09T15:38:50,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34975 2024-11-09T15:38:50,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34975 2024-11-09T15:38:50,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-09T15:38:50,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-09T15:38:50,563 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da6b20b51b98:0 server-side Connection retries=9 2024-11-09T15:38:50,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T15:38:50,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T15:38:50,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T15:38:50,569 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T15:38:50,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T15:38:50,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45207 2024-11-09T15:38:50,575 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45207 connecting to ZooKeeper ensemble=127.0.0.1:53003 2024-11-09T15:38:50,576 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:50,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:50,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452070x0, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T15:38:50,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45207-0x1018daf68880001 connected 2024-11-09T15:38:50,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T15:38:50,594 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T15:38:50,602 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T15:38:50,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T15:38:50,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T15:38:50,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45207 2024-11-09T15:38:50,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45207 2024-11-09T15:38:50,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45207 2024-11-09T15:38:50,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45207 2024-11-09T15:38:50,613 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45207 2024-11-09T15:38:50,629 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da6b20b51b98:34975 2024-11-09T15:38:50,630 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da6b20b51b98,34975,1731166729839 2024-11-09T15:38:50,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T15:38:50,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T15:38:50,640 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da6b20b51b98,34975,1731166729839 2024-11-09T15:38:50,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T15:38:50,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:50,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:50,665 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T15:38:50,667 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da6b20b51b98,34975,1731166729839 from backup master directory 2024-11-09T15:38:50,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da6b20b51b98,34975,1731166729839 2024-11-09T15:38:50,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T15:38:50,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T15:38:50,670 WARN [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T15:38:50,671 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da6b20b51b98,34975,1731166729839 2024-11-09T15:38:50,673 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T15:38:50,674 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T15:38:50,733 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/hbase.id] with ID: 3ff26ad6-1938-46dc-ba5d-bc9de655af94 2024-11-09T15:38:50,734 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/.tmp/hbase.id 2024-11-09T15:38:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741826_1002 (size=42) 2024-11-09T15:38:51,148 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/.tmp/hbase.id]:[hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/hbase.id] 2024-11-09T15:38:51,193 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T15:38:51,198 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T15:38:51,217 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-09T15:38:51,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:51,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:51,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741827_1003 (size=196) 2024-11-09T15:38:51,654 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:38:51,657 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T15:38:51,674 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T15:38:51,679 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T15:38:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741828_1004 (size=1189) 2024-11-09T15:38:52,130 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store 2024-11-09T15:38:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741829_1005 (size=34) 2024-11-09T15:38:52,156 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T15:38:52,159 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:38:52,161 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T15:38:52,161 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:38:52,161 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:38:52,163 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T15:38:52,163 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:38:52,163 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:38:52,164 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731166732160Disabling compacts and flushes for region at 1731166732160Disabling writes for close at 1731166732163 (+3 ms)Writing region close event to WAL at 1731166732163Closed at 1731166732163 2024-11-09T15:38:52,167 WARN [master/da6b20b51b98:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/.initializing 2024-11-09T15:38:52,167 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/WALs/da6b20b51b98,34975,1731166729839 2024-11-09T15:38:52,176 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T15:38:52,193 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da6b20b51b98%2C34975%2C1731166729839, suffix=, logDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/WALs/da6b20b51b98,34975,1731166729839, archiveDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/oldWALs, maxLogs=10 2024-11-09T15:38:52,219 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/WALs/da6b20b51b98,34975,1731166729839/da6b20b51b98%2C34975%2C1731166729839.1731166732198, exclude list is [], retry=0 2024-11-09T15:38:52,239 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-936698ce-4a19-44b2-8dfc-1f99bfbd77cf,DISK] 2024-11-09T15:38:52,242 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T15:38:52,280 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/WALs/da6b20b51b98,34975,1731166729839/da6b20b51b98%2C34975%2C1731166729839.1731166732198 2024-11-09T15:38:52,281 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43781:43781)] 2024-11-09T15:38:52,282 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:38:52,282 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:38:52,285 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,287 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T15:38:52,358 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:52,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:38:52,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T15:38:52,366 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:52,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:38:52,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T15:38:52,370 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:52,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:38:52,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T15:38:52,375 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:52,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:38:52,377 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,380 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,382 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,387 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,388 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,391 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T15:38:52,395 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T15:38:52,400 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:38:52,401 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65962764, jitterRate=-0.0170782208442688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T15:38:52,408 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731166732300Initializing all the Stores at 1731166732303 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166732303Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166732304 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166732304Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166732304Cleaning up temporary data from old regions at 1731166732388 (+84 ms)Region opened successfully at 1731166732407 (+19 ms) 2024-11-09T15:38:52,409 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T15:38:52,446 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ebcb50f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da6b20b51b98/172.17.0.2:0 2024-11-09T15:38:52,481 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T15:38:52,494 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T15:38:52,494 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T15:38:52,497 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T15:38:52,499 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T15:38:52,504 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T15:38:52,504 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T15:38:52,532 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T15:38:52,541 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T15:38:52,544 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T15:38:52,547 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T15:38:52,548 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T15:38:52,550 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T15:38:52,553 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T15:38:52,556 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T15:38:52,558 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T15:38:52,559 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T15:38:52,561 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T15:38:52,580 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T15:38:52,582 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T15:38:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T15:38:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T15:38:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,589 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da6b20b51b98,34975,1731166729839, sessionid=0x1018daf68880000, setting cluster-up flag (Was=false) 2024-11-09T15:38:52,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,611 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T15:38:52,613 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da6b20b51b98,34975,1731166729839 2024-11-09T15:38:52,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:38:52,625 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T15:38:52,627 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da6b20b51b98,34975,1731166729839 2024-11-09T15:38:52,633 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T15:38:52,710 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T15:38:52,718 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(746): ClusterId : 3ff26ad6-1938-46dc-ba5d-bc9de655af94 2024-11-09T15:38:52,722 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T15:38:52,722 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T15:38:52,728 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T15:38:52,729 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T15:38:52,729 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T15:38:52,733 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T15:38:52,734 DEBUG [RS:0;da6b20b51b98:45207 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1187f006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da6b20b51b98/172.17.0.2:0 2024-11-09T15:38:52,734 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da6b20b51b98,34975,1731166729839 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T15:38:52,741 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da6b20b51b98:0, corePoolSize=5, maxPoolSize=5 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da6b20b51b98:0, corePoolSize=5, maxPoolSize=5 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da6b20b51b98:0, corePoolSize=5, maxPoolSize=5 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da6b20b51b98:0, corePoolSize=5, maxPoolSize=5 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da6b20b51b98:0, corePoolSize=10, maxPoolSize=10 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da6b20b51b98:0, corePoolSize=2, maxPoolSize=2 2024-11-09T15:38:52,742 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,744 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731166762744 2024-11-09T15:38:52,746 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T15:38:52,747 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T15:38:52,748 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T15:38:52,748 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T15:38:52,752 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T15:38:52,752 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T15:38:52,753 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T15:38:52,753 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T15:38:52,754 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da6b20b51b98:45207 2024-11-09T15:38:52,754 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,756 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T15:38:52,756 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:52,757 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T15:38:52,757 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T15:38:52,758 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T15:38:52,759 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T15:38:52,759 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T15:38:52,759 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T15:38:52,760 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T15:38:52,760 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T15:38:52,763 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(2659): reportForDuty to master=da6b20b51b98,34975,1731166729839 with port=45207, startcode=1731166730521 2024-11-09T15:38:52,763 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.large.0-1731166732761,5,FailOnTimeoutGroup] 2024-11-09T15:38:52,763 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.small.0-1731166732763,5,FailOnTimeoutGroup] 2024-11-09T15:38:52,764 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,764 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T15:38:52,766 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,766 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741831_1007 (size=1321) 2024-11-09T15:38:52,776 DEBUG [RS:0;da6b20b51b98:45207 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T15:38:52,816 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T15:38:52,823 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,826 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34975 {}] master.ServerManager(517): Registering regionserver=da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,843 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:38:52,843 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34413 2024-11-09T15:38:52,843 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T15:38:52,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T15:38:52,849 DEBUG [RS:0;da6b20b51b98:45207 {}] zookeeper.ZKUtil(111): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,849 WARN [RS:0;da6b20b51b98:45207 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T15:38:52,849 INFO [RS:0;da6b20b51b98:45207 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T15:38:52,849 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da6b20b51b98,45207,1731166730521] 2024-11-09T15:38:52,878 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T15:38:52,893 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T15:38:52,898 INFO [RS:0;da6b20b51b98:45207 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T15:38:52,898 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,899 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T15:38:52,905 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T15:38:52,907 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,907 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,907 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,907 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da6b20b51b98:0, corePoolSize=2, maxPoolSize=2 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,908 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,909 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,909 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,909 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da6b20b51b98:0, corePoolSize=1, maxPoolSize=1 2024-11-09T15:38:52,909 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da6b20b51b98:0, corePoolSize=3, maxPoolSize=3 2024-11-09T15:38:52,909 DEBUG [RS:0;da6b20b51b98:45207 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da6b20b51b98:0, corePoolSize=3, maxPoolSize=3 2024-11-09T15:38:52,911 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,911 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,911 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,912 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,912 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,912 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,45207,1731166730521-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T15:38:52,940 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T15:38:52,942 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,45207,1731166730521-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,943 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,943 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.Replication(171): da6b20b51b98,45207,1731166730521 started 2024-11-09T15:38:52,968 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:38:52,969 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1482): Serving as da6b20b51b98,45207,1731166730521, RpcServer on da6b20b51b98/172.17.0.2:45207, sessionid=0x1018daf68880001 2024-11-09T15:38:52,970 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T15:38:52,970 DEBUG [RS:0;da6b20b51b98:45207 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,970 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da6b20b51b98,45207,1731166730521' 2024-11-09T15:38:52,970 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T15:38:52,971 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T15:38:52,972 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T15:38:52,972 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T15:38:52,973 DEBUG [RS:0;da6b20b51b98:45207 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da6b20b51b98,45207,1731166730521 2024-11-09T15:38:52,973 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da6b20b51b98,45207,1731166730521' 2024-11-09T15:38:52,973 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T15:38:52,973 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T15:38:52,974 DEBUG [RS:0;da6b20b51b98:45207 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T15:38:52,974 INFO [RS:0;da6b20b51b98:45207 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T15:38:52,974 INFO [RS:0;da6b20b51b98:45207 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T15:38:53,172 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T15:38:53,172 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:38:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741832_1008 (size=32) 2024-11-09T15:38:53,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:38:53,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T15:38:53,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T15:38:53,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:53,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:38:53,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T15:38:53,594 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T15:38:53,594 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:53,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:38:53,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T15:38:53,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T15:38:53,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:53,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:38:53,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T15:38:53,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T15:38:53,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:38:53,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:38:53,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T15:38:53,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740 2024-11-09T15:38:53,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740 2024-11-09T15:38:53,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T15:38:53,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T15:38:53,608 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T15:38:53,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T15:38:53,614 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:38:53,615 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59097669, jitterRate=-0.11937610805034637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T15:38:53,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731166733585Initializing all the Stores at 1731166733587 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166733587Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166733587Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166733587Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166733587Cleaning up temporary data from old regions at 1731166733607 (+20 ms)Region opened successfully at 1731166733617 (+10 ms) 2024-11-09T15:38:53,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T15:38:53,617 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T15:38:53,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T15:38:53,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T15:38:53,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T15:38:53,618 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T15:38:53,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731166733617Disabling compacts and flushes for region at 1731166733617Disabling writes for close at 1731166733617Writing region close event to WAL at 1731166733618 (+1 ms)Closed at 1731166733618 2024-11-09T15:38:53,622 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T15:38:53,622 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T15:38:53,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T15:38:53,637 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T15:38:53,640 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T15:38:53,791 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:54,792 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:55,792 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:56,793 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:57,794 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:58,795 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:38:59,795 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:00,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T15:39:00,249 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:00,796 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:01,797 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:02,797 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:03,798 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:04,799 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:05,799 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:06,800 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:07,801 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:08,801 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:09,802 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:10,803 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:11,803 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:12,804 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:13,805 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:14,806 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:15,806 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:16,807 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:17,808 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:18,808 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:18,928 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T15:39:19,809 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:20,810 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:21,810 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:22,811 WARN [da6b20b51b98:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T15:39:22,980 INFO [RS:0;da6b20b51b98:45207 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T15:39:22,985 INFO [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da6b20b51b98%2C45207%2C1731166730521, suffix=, logDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521, archiveDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/oldWALs, maxLogs=32 2024-11-09T15:39:23,002 DEBUG [RS:0;da6b20b51b98:45207 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521/da6b20b51b98%2C45207%2C1731166730521.1731166762987, exclude list is [], retry=0 2024-11-09T15:39:23,007 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-936698ce-4a19-44b2-8dfc-1f99bfbd77cf,DISK] 2024-11-09T15:39:23,010 INFO [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521/da6b20b51b98%2C45207%2C1731166730521.1731166762987 2024-11-09T15:39:23,011 DEBUG [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43781:43781)] 2024-11-09T15:39:23,063 DEBUG [da6b20b51b98:34975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T15:39:23,076 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:23,083 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da6b20b51b98,45207,1731166730521, state=OPENING 2024-11-09T15:39:23,088 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T15:39:23,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:39:23,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:39:23,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T15:39:23,091 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T15:39:23,092 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T15:39:23,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:23,270 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T15:39:23,274 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55259, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T15:39:23,289 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T15:39:23,290 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T15:39:23,290 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T15:39:23,294 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da6b20b51b98%2C45207%2C1731166730521.meta, suffix=.meta, logDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521, archiveDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/oldWALs, maxLogs=32 2024-11-09T15:39:23,311 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521/da6b20b51b98%2C45207%2C1731166730521.meta.1731166763296.meta, exclude list is [], retry=0 2024-11-09T15:39:23,315 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-936698ce-4a19-44b2-8dfc-1f99bfbd77cf,DISK] 2024-11-09T15:39:23,318 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/WALs/da6b20b51b98,45207,1731166730521/da6b20b51b98%2C45207%2C1731166730521.meta.1731166763296.meta 2024-11-09T15:39:23,318 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43781:43781)] 2024-11-09T15:39:23,319 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:23,321 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T15:39:23,323 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T15:39:23,328 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T15:39:23,332 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T15:39:23,333 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:23,333 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T15:39:23,333 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T15:39:23,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T15:39:23,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T15:39:23,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:23,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:39:23,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T15:39:23,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T15:39:23,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:23,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:39:23,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T15:39:23,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T15:39:23,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:23,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:39:23,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T15:39:23,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T15:39:23,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:23,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T15:39:23,346 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T15:39:23,348 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740 2024-11-09T15:39:23,351 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740 2024-11-09T15:39:23,353 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T15:39:23,353 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T15:39:23,354 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T15:39:23,357 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T15:39:23,359 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62553075, jitterRate=-0.06788654625415802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T15:39:23,359 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T15:39:23,361 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731166763334Writing region info on filesystem at 1731166763334Initializing all the Stores at 1731166763336 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166763336Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166763336Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166763336Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731166763336Cleaning up temporary data from old regions at 1731166763354 (+18 ms)Running coprocessor post-open hooks at 1731166763359 (+5 ms)Region opened successfully at 1731166763361 (+2 ms) 2024-11-09T15:39:23,369 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731166763261 2024-11-09T15:39:23,381 DEBUG [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T15:39:23,381 INFO [RS_OPEN_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T15:39:23,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:23,385 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da6b20b51b98,45207,1731166730521, state=OPEN 2024-11-09T15:39:23,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T15:39:23,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T15:39:23,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T15:39:23,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T15:39:23,390 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:23,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T15:39:23,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da6b20b51b98,45207,1731166730521 in 296 msec 2024-11-09T15:39:23,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T15:39:23,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.7690 sec 2024-11-09T15:39:23,405 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T15:39:23,405 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T15:39:23,427 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T15:39:23,428 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da6b20b51b98,45207,1731166730521, seqNum=-1] 2024-11-09T15:39:23,452 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T15:39:23,455 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51757, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T15:39:23,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8120 sec 2024-11-09T15:39:23,476 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731166763475, completionTime=-1 2024-11-09T15:39:23,479 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T15:39:23,479 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T15:39:23,507 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-09T15:39:23,507 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731166823507 2024-11-09T15:39:23,507 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731166883507 2024-11-09T15:39:23,508 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-09T15:39:23,510 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,510 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,510 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,512 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da6b20b51b98:34975, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,513 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,513 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,519 DEBUG [master/da6b20b51b98:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T15:39:23,542 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 32.870sec 2024-11-09T15:39:23,544 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T15:39:23,545 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T15:39:23,547 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T15:39:23,547 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T15:39:23,547 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T15:39:23,548 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T15:39:23,549 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T15:39:23,557 DEBUG [master/da6b20b51b98:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T15:39:23,558 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T15:39:23,559 INFO [master/da6b20b51b98:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da6b20b51b98,34975,1731166729839-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T15:39:23,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,654 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T15:39:23,654 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T15:39:23,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da6b20b51b98,34975,-1 for getting cluster id 2024-11-09T15:39:23,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T15:39:23,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3ff26ad6-1938-46dc-ba5d-bc9de655af94' 2024-11-09T15:39:23,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T15:39:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3ff26ad6-1938-46dc-ba5d-bc9de655af94" 2024-11-09T15:39:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b758490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da6b20b51b98,34975,-1] 2024-11-09T15:39:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T15:39:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:39:23,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T15:39:23,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T15:39:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da6b20b51b98,45207,1731166730521, seqNum=-1] 2024-11-09T15:39:23,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T15:39:23,694 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T15:39:23,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da6b20b51b98,34975,1731166729839 2024-11-09T15:39:23,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da6b20b51b98,34975,-1 for getting cluster id 2024-11-09T15:39:23,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T15:39:23,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3ff26ad6-1938-46dc-ba5d-bc9de655af94' 2024-11-09T15:39:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T15:39:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3ff26ad6-1938-46dc-ba5d-bc9de655af94" 2024-11-09T15:39:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd4d5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da6b20b51b98,34975,-1] 2024-11-09T15:39:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T15:39:23,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:39:23,724 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T15:39:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467ca67d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T15:39:23,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=146, ProcessCount=11, AvailableMemoryMB=7800 2024-11-09T15:39:23,745 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T15:39:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is da6b20b51b98,34975,1731166729839 2024-11-09T15:39:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ef18a0 2024-11-09T15:39:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T15:39:23,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T15:39:23,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-09T15:39:23,798 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:23,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-09T15:39:23,801 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:23,804 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T15:39:23,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741835_1011 (size=428) 2024-11-09T15:39:24,227 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45ba5a0d25f5f40ef5c8117cd0fa3177, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:24,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741836_1012 (size=79) 2024-11-09T15:39:24,638 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:24,638 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 45ba5a0d25f5f40ef5c8117cd0fa3177, disabling compactions & flushes 2024-11-09T15:39:24,639 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:24,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:24,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. after waiting 0 ms 2024-11-09T15:39:24,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:24,639 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:24,639 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45ba5a0d25f5f40ef5c8117cd0fa3177: Waiting for close lock at 1731166764638Disabling compacts and flushes for region at 1731166764638Disabling writes for close at 1731166764639 (+1 ms)Writing region close event to WAL at 1731166764639Closed at 1731166764639 2024-11-09T15:39:24,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:24,647 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731166764642"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166764642"}]},"ts":"1731166764642"} 2024-11-09T15:39:24,653 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:24,655 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:24,658 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166764655"}]},"ts":"1731166764655"} 2024-11-09T15:39:24,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-09T15:39:24,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=45ba5a0d25f5f40ef5c8117cd0fa3177, ASSIGN}] 2024-11-09T15:39:24,667 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=45ba5a0d25f5f40ef5c8117cd0fa3177, ASSIGN 2024-11-09T15:39:24,668 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=45ba5a0d25f5f40ef5c8117cd0fa3177, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T15:39:24,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=45ba5a0d25f5f40ef5c8117cd0fa3177, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:24,824 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=45ba5a0d25f5f40ef5c8117cd0fa3177, ASSIGN because future has completed 2024-11-09T15:39:24,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45ba5a0d25f5f40ef5c8117cd0fa3177, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:24,987 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:24,987 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 45ba5a0d25f5f40ef5c8117cd0fa3177, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:24,988 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,988 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:24,988 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,988 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,991 INFO [StoreOpener-45ba5a0d25f5f40ef5c8117cd0fa3177-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,993 INFO [StoreOpener-45ba5a0d25f5f40ef5c8117cd0fa3177-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ba5a0d25f5f40ef5c8117cd0fa3177 columnFamilyName testFamily 2024-11-09T15:39:24,993 DEBUG [StoreOpener-45ba5a0d25f5f40ef5c8117cd0fa3177-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:24,994 INFO [StoreOpener-45ba5a0d25f5f40ef5c8117cd0fa3177-1 {}] regionserver.HStore(327): Store=45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:24,995 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,996 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,997 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,998 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:24,998 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:25,001 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:25,005 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:25,006 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 45ba5a0d25f5f40ef5c8117cd0fa3177; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58992713, jitterRate=-0.12094007432460785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:25,007 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:39:25,008 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 45ba5a0d25f5f40ef5c8117cd0fa3177: Running coprocessor pre-open hook at 1731166764988Writing region info on filesystem at 1731166764988Initializing all the Stores at 1731166764990 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166764990Cleaning up temporary data from old regions at 1731166764998 (+8 ms)Running coprocessor post-open hooks at 1731166765007 (+9 ms)Region opened successfully at 1731166765008 (+1 ms) 2024-11-09T15:39:25,010 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177., pid=6, masterSystemTime=1731166764979 2024-11-09T15:39:25,014 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:25,014 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:39:25,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=45ba5a0d25f5f40ef5c8117cd0fa3177, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:25,019 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45ba5a0d25f5f40ef5c8117cd0fa3177, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:25,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T15:39:25,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 45ba5a0d25f5f40ef5c8117cd0fa3177, server=da6b20b51b98,45207,1731166730521 in 196 msec 2024-11-09T15:39:25,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T15:39:25,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=45ba5a0d25f5f40ef5c8117cd0fa3177, ASSIGN in 361 msec 2024-11-09T15:39:25,030 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:25,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166765031"}]},"ts":"1731166765031"} 2024-11-09T15:39:25,034 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-09T15:39:25,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:25,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2480 sec 2024-11-09T15:39:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T15:39:26,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-09T15:39:26,840 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-09T15:39:26,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:26,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-09T15:39:26,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:26,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-09T15:39:26,851 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T15:39:26,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da6b20b51b98,45207,1731166730521, seqNum=-1] 2024-11-09T15:39:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T15:39:26,855 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T15:39:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:26,877 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:26,881 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-09T15:39:26,886 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T15:39:26,891 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T15:39:26,893 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T15:39:26,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-1567105645004455928 2024-11-09T15:39:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166769893, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:26,924 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-09T15:39:26,925 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-09T15:39:26,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:26,927 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:26,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-1567105645004455926 2024-11-09T15:39:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166769927, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:26,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:26,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-1567105645004455925 2024-11-09T15:39:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166769932, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:26,937 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:26,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-1567105645004455924 2024-11-09T15:39:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166769936, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:26,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:26,957 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1700936484_22 at /127.0.0.1:52620 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=146 (was 146), ProcessCount=11 (was 11), AvailableMemoryMB=7752 (was 7800) 2024-11-09T15:39:26,966 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=146, ProcessCount=11, AvailableMemoryMB=7751 2024-11-09T15:39:26,970 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:26,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-09T15:39:26,974 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:26,974 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:26,974 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-09T15:39:26,976 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T15:39:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741837_1013 (size=414) 2024-11-09T15:39:27,389 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab830ca42595810345ca62bc8a3c2219, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741838_1014 (size=65) 2024-11-09T15:39:27,398 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:27,399 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ab830ca42595810345ca62bc8a3c2219, disabling compactions & flushes 2024-11-09T15:39:27,399 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,399 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,399 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. after waiting 0 ms 2024-11-09T15:39:27,399 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,399 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,399 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab830ca42595810345ca62bc8a3c2219: Waiting for close lock at 1731166767399Disabling compacts and flushes for region at 1731166767399Disabling writes for close at 1731166767399Writing region close event to WAL at 1731166767399Closed at 1731166767399 2024-11-09T15:39:27,402 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:27,402 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731166767402"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166767402"}]},"ts":"1731166767402"} 2024-11-09T15:39:27,406 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:27,408 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:27,408 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166767408"}]},"ts":"1731166767408"} 2024-11-09T15:39:27,412 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-09T15:39:27,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ab830ca42595810345ca62bc8a3c2219, ASSIGN}] 2024-11-09T15:39:27,414 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ab830ca42595810345ca62bc8a3c2219, ASSIGN 2024-11-09T15:39:27,416 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ab830ca42595810345ca62bc8a3c2219, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:27,567 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ab830ca42595810345ca62bc8a3c2219, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:27,572 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ab830ca42595810345ca62bc8a3c2219, ASSIGN because future has completed 2024-11-09T15:39:27,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab830ca42595810345ca62bc8a3c2219, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:27,733 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ab830ca42595810345ca62bc8a3c2219, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,737 INFO [StoreOpener-ab830ca42595810345ca62bc8a3c2219-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,739 INFO [StoreOpener-ab830ca42595810345ca62bc8a3c2219-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab830ca42595810345ca62bc8a3c2219 columnFamilyName testFamily 2024-11-09T15:39:27,739 DEBUG [StoreOpener-ab830ca42595810345ca62bc8a3c2219-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:27,740 INFO [StoreOpener-ab830ca42595810345ca62bc8a3c2219-1 {}] regionserver.HStore(327): Store=ab830ca42595810345ca62bc8a3c2219/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:27,740 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,743 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,746 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:27,750 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened ab830ca42595810345ca62bc8a3c2219; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67408927, jitterRate=0.004471287131309509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:27,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:39:27,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ab830ca42595810345ca62bc8a3c2219: Running coprocessor pre-open hook at 1731166767734Writing region info on filesystem at 1731166767734Initializing all the Stores at 1731166767736 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166767736Cleaning up temporary data from old regions at 1731166767743 (+7 ms)Running coprocessor post-open hooks at 1731166767750 (+7 ms)Region opened successfully at 1731166767751 (+1 ms) 2024-11-09T15:39:27,752 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., pid=9, masterSystemTime=1731166767726 2024-11-09T15:39:27,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:39:27,758 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ab830ca42595810345ca62bc8a3c2219, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:27,761 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab830ca42595810345ca62bc8a3c2219, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:27,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-09T15:39:27,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ab830ca42595810345ca62bc8a3c2219, server=da6b20b51b98,45207,1731166730521 in 191 msec 2024-11-09T15:39:27,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T15:39:27,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ab830ca42595810345ca62bc8a3c2219, ASSIGN in 354 msec 2024-11-09T15:39:27,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:27,771 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166767771"}]},"ts":"1731166767771"} 2024-11-09T15:39:27,774 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-09T15:39:27,775 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:27,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 806 msec 2024-11-09T15:39:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T15:39:27,988 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-09T15:39:27,988 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-09T15:39:27,989 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:27,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-09T15:39:27,993 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:27,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-09T15:39:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:28,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:28,010 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-09T15:39:28,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T15:39:28,015 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-09T15:39:28,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:39:28,893 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-09T15:39:28,895 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T15:39:28,895 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-09T15:39:30,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T15:39:30,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:30,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T15:39:30,249 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T15:39:30,250 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-09T15:39:30,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:30,251 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-09T15:39:30,251 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:34,029 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-09T15:39:34,030 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-09T15:39:34,030 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:39:34,042 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 146), ProcessCount=11 (was 11), AvailableMemoryMB=8510 (was 7751) - AvailableMemoryMB LEAK? - 2024-11-09T15:39:34,052 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=8510 2024-11-09T15:39:34,055 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:34,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-09T15:39:34,059 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:34,059 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:34,060 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-09T15:39:34,061 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T15:39:34,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741839_1015 (size=412) 2024-11-09T15:39:34,474 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 55ba359dcd9c97c003d064a2efb76221, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:34,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741840_1016 (size=63) 2024-11-09T15:39:34,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 55ba359dcd9c97c003d064a2efb76221, disabling compactions & flushes 2024-11-09T15:39:34,890 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. after waiting 0 ms 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:34,890 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:34,890 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 55ba359dcd9c97c003d064a2efb76221: Waiting for close lock at 1731166774890Disabling compacts and flushes for region at 1731166774890Disabling writes for close at 1731166774890Writing region close event to WAL at 1731166774890Closed at 1731166774890 2024-11-09T15:39:34,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:34,893 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731166774893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166774893"}]},"ts":"1731166774893"} 2024-11-09T15:39:34,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:34,898 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:34,898 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166774898"}]},"ts":"1731166774898"} 2024-11-09T15:39:34,901 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-09T15:39:34,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=55ba359dcd9c97c003d064a2efb76221, ASSIGN}] 2024-11-09T15:39:34,904 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=55ba359dcd9c97c003d064a2efb76221, ASSIGN 2024-11-09T15:39:34,905 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=55ba359dcd9c97c003d064a2efb76221, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:35,056 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=55ba359dcd9c97c003d064a2efb76221, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:35,060 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=55ba359dcd9c97c003d064a2efb76221, ASSIGN because future has completed 2024-11-09T15:39:35,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55ba359dcd9c97c003d064a2efb76221, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T15:39:35,220 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:35,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 55ba359dcd9c97c003d064a2efb76221, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:35,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:35,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,223 INFO [StoreOpener-55ba359dcd9c97c003d064a2efb76221-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,226 INFO [StoreOpener-55ba359dcd9c97c003d064a2efb76221-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55ba359dcd9c97c003d064a2efb76221 columnFamilyName testFamily 2024-11-09T15:39:35,226 DEBUG [StoreOpener-55ba359dcd9c97c003d064a2efb76221-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:35,226 INFO [StoreOpener-55ba359dcd9c97c003d064a2efb76221-1 {}] regionserver.HStore(327): Store=55ba359dcd9c97c003d064a2efb76221/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:35,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,228 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,228 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:35,235 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 55ba359dcd9c97c003d064a2efb76221; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74509643, jitterRate=0.11028020083904266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:35,235 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:39:35,236 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 55ba359dcd9c97c003d064a2efb76221: Running coprocessor pre-open hook at 1731166775221Writing region info on filesystem at 1731166775221Initializing all the Stores at 1731166775223 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166775223Cleaning up temporary data from old regions at 1731166775229 (+6 ms)Running coprocessor post-open hooks at 1731166775235 (+6 ms)Region opened successfully at 1731166775236 (+1 ms) 2024-11-09T15:39:35,238 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221., pid=12, masterSystemTime=1731166775214 2024-11-09T15:39:35,241 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:35,241 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:39:35,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=55ba359dcd9c97c003d064a2efb76221, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:35,246 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55ba359dcd9c97c003d064a2efb76221, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:35,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T15:39:35,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 55ba359dcd9c97c003d064a2efb76221, server=da6b20b51b98,45207,1731166730521 in 187 msec 2024-11-09T15:39:35,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-09T15:39:35,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=55ba359dcd9c97c003d064a2efb76221, ASSIGN in 351 msec 2024-11-09T15:39:35,256 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:35,257 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166775256"}]},"ts":"1731166775256"} 2024-11-09T15:39:35,259 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-09T15:39:35,261 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:35,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2060 sec 2024-11-09T15:39:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T15:39:37,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-09T15:39:37,089 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-09T15:39:37,089 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:37,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-09T15:39:37,095 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:37,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-09T15:39:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:37,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:37,120 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-09T15:39:37,125 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T15:39:37,127 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T15:39:37,129 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T15:39:37,130 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-09T15:39:40,738 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:39:40,758 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-09T15:39:40,758 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:40,897 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-09T15:39:43,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455915' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T15:39:43,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46936 deadline: 1731166789133, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455915' does not exist 2024-11-09T15:39:43,137 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to da6b20b51b98,45207,1731166730521 for closing scanner id = -1567105645004455915 for 55ba359dcd9c97c003d064a2efb76221 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455915' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T15:39:43,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=8271 (was 8510) 2024-11-09T15:39:43,148 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=8271 2024-11-09T15:39:43,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-09T15:39:43,154 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:43,154 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:43,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-09T15:39:43,155 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T15:39:43,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741841_1017 (size=423) 2024-11-09T15:39:43,568 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd1cf2ea3c38eda30f839dfd5f4eb192, NAME => 'testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:43,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741842_1018 (size=74) 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing bd1cf2ea3c38eda30f839dfd5f4eb192, disabling compactions & flushes 2024-11-09T15:39:43,581 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. after waiting 0 ms 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,581 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,581 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd1cf2ea3c38eda30f839dfd5f4eb192: Waiting for close lock at 1731166783581Disabling compacts and flushes for region at 1731166783581Disabling writes for close at 1731166783581Writing region close event to WAL at 1731166783581Closed at 1731166783581 2024-11-09T15:39:43,584 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:43,584 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731166783584"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166783584"}]},"ts":"1731166783584"} 2024-11-09T15:39:43,588 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:43,590 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:43,591 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166783590"}]},"ts":"1731166783590"} 2024-11-09T15:39:43,593 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-09T15:39:43,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bd1cf2ea3c38eda30f839dfd5f4eb192, ASSIGN}] 2024-11-09T15:39:43,596 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bd1cf2ea3c38eda30f839dfd5f4eb192, ASSIGN 2024-11-09T15:39:43,597 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bd1cf2ea3c38eda30f839dfd5f4eb192, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-09T15:39:43,748 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=bd1cf2ea3c38eda30f839dfd5f4eb192, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:43,752 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bd1cf2ea3c38eda30f839dfd5f4eb192, ASSIGN because future has completed 2024-11-09T15:39:43,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd1cf2ea3c38eda30f839dfd5f4eb192, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:43,912 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,913 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => bd1cf2ea3c38eda30f839dfd5f4eb192, NAME => 'testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:43,913 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,913 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:43,913 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,913 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,916 INFO [StoreOpener-bd1cf2ea3c38eda30f839dfd5f4eb192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,920 INFO [StoreOpener-bd1cf2ea3c38eda30f839dfd5f4eb192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd1cf2ea3c38eda30f839dfd5f4eb192 columnFamilyName testFamily 2024-11-09T15:39:43,920 DEBUG [StoreOpener-bd1cf2ea3c38eda30f839dfd5f4eb192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:43,921 INFO [StoreOpener-bd1cf2ea3c38eda30f839dfd5f4eb192-1 {}] regionserver.HStore(327): Store=bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:43,921 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,922 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,923 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,924 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,924 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,926 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,930 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:43,930 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened bd1cf2ea3c38eda30f839dfd5f4eb192; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69266600, jitterRate=0.032152771949768066}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:43,930 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:39:43,931 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for bd1cf2ea3c38eda30f839dfd5f4eb192: Running coprocessor pre-open hook at 1731166783914Writing region info on filesystem at 1731166783914Initializing all the Stores at 1731166783915 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166783916 (+1 ms)Cleaning up temporary data from old regions at 1731166783924 (+8 ms)Running coprocessor post-open hooks at 1731166783930 (+6 ms)Region opened successfully at 1731166783931 (+1 ms) 2024-11-09T15:39:43,932 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192., pid=15, masterSystemTime=1731166783905 2024-11-09T15:39:43,936 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,936 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:39:43,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=bd1cf2ea3c38eda30f839dfd5f4eb192, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:43,939 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd1cf2ea3c38eda30f839dfd5f4eb192, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:43,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-09T15:39:43,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure bd1cf2ea3c38eda30f839dfd5f4eb192, server=da6b20b51b98,45207,1731166730521 in 189 msec 2024-11-09T15:39:43,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T15:39:43,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=bd1cf2ea3c38eda30f839dfd5f4eb192, ASSIGN in 350 msec 2024-11-09T15:39:43,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:43,949 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166783948"}]},"ts":"1731166783948"} 2024-11-09T15:39:43,951 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-09T15:39:43,952 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:43,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 802 msec 2024-11-09T15:39:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T15:39:44,169 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-09T15:39:44,169 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-09T15:39:44,169 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:44,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-09T15:39:44,174 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:44,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-09T15:39:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:44,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:44,188 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-09T15:39:44,190 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T15:39:44,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T15:39:44,193 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T15:39:44,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-1567105645004455911 2024-11-09T15:39:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166787193, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:44,197 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-09T15:39:44,198 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-09T15:39:44,199 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:44,200 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:44,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-1567105645004455909 2024-11-09T15:39:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166787199, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:44,204 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:44,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-1567105645004455908 2024-11-09T15:39:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166787203, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:44,207 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:44,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-1567105645004455907 2024-11-09T15:39:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:46936 deadline: 1731166787207, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T15:39:44,211 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T15:39:44,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1700936484_22 at /127.0.0.1:51498 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=8260 (was 8271) 2024-11-09T15:39:44,229 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=8259 2024-11-09T15:39:44,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-09T15:39:44,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:44,234 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:44,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-09T15:39:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T15:39:44,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:44,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741843_1019 (size=418) 2024-11-09T15:39:44,647 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8812061f09cf80dc49527a5a0a9f5a00, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:44,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741844_1020 (size=69) 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 8812061f09cf80dc49527a5a0a9f5a00, disabling compactions & flushes 2024-11-09T15:39:45,057 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. after waiting 0 ms 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,057 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,057 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8812061f09cf80dc49527a5a0a9f5a00: Waiting for close lock at 1731166785057Disabling compacts and flushes for region at 1731166785057Disabling writes for close at 1731166785057Writing region close event to WAL at 1731166785057Closed at 1731166785057 2024-11-09T15:39:45,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:45,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731166785059"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166785059"}]},"ts":"1731166785059"} 2024-11-09T15:39:45,063 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:45,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:45,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166785064"}]},"ts":"1731166785064"} 2024-11-09T15:39:45,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-09T15:39:45,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8812061f09cf80dc49527a5a0a9f5a00, ASSIGN}] 2024-11-09T15:39:45,069 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8812061f09cf80dc49527a5a0a9f5a00, ASSIGN 2024-11-09T15:39:45,070 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8812061f09cf80dc49527a5a0a9f5a00, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:45,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8812061f09cf80dc49527a5a0a9f5a00, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:45,224 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8812061f09cf80dc49527a5a0a9f5a00, ASSIGN because future has completed 2024-11-09T15:39:45,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8812061f09cf80dc49527a5a0a9f5a00, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T15:39:45,383 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,384 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 8812061f09cf80dc49527a5a0a9f5a00, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:45,384 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,384 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:45,384 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,384 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,386 INFO [StoreOpener-8812061f09cf80dc49527a5a0a9f5a00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,388 INFO [StoreOpener-8812061f09cf80dc49527a5a0a9f5a00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8812061f09cf80dc49527a5a0a9f5a00 columnFamilyName testFamily 2024-11-09T15:39:45,389 DEBUG [StoreOpener-8812061f09cf80dc49527a5a0a9f5a00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:45,389 INFO [StoreOpener-8812061f09cf80dc49527a5a0a9f5a00-1 {}] regionserver.HStore(327): Store=8812061f09cf80dc49527a5a0a9f5a00/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:45,389 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,391 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,391 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,392 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,392 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,394 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,398 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:45,398 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 8812061f09cf80dc49527a5a0a9f5a00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64685510, jitterRate=-0.03611078858375549}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:45,399 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:39:45,399 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 8812061f09cf80dc49527a5a0a9f5a00: Running coprocessor pre-open hook at 1731166785384Writing region info on filesystem at 1731166785384Initializing all the Stores at 1731166785386 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166785386Cleaning up temporary data from old regions at 1731166785392 (+6 ms)Running coprocessor post-open hooks at 1731166785399 (+7 ms)Region opened successfully at 1731166785399 2024-11-09T15:39:45,401 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., pid=18, masterSystemTime=1731166785377 2024-11-09T15:39:45,404 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,404 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:39:45,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8812061f09cf80dc49527a5a0a9f5a00, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:45,408 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8812061f09cf80dc49527a5a0a9f5a00, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:45,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T15:39:45,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 8812061f09cf80dc49527a5a0a9f5a00, server=da6b20b51b98,45207,1731166730521 in 185 msec 2024-11-09T15:39:45,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-09T15:39:45,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8812061f09cf80dc49527a5a0a9f5a00, ASSIGN in 345 msec 2024-11-09T15:39:45,417 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:45,417 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166785417"}]},"ts":"1731166785417"} 2024-11-09T15:39:45,419 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-09T15:39:45,420 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:45,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1890 sec 2024-11-09T15:39:46,898 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-09T15:39:46,899 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-09T15:39:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T15:39:47,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-09T15:39:47,259 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-09T15:39:47,259 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:47,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-09T15:39:47,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:47,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-09T15:39:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:47,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:47,280 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-09T15:39:47,283 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T15:39:47,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T15:39:48,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T15:39:48,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-09T15:39:48,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:39:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-09T15:39:48,929 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T15:39:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:49,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T15:39:49,415 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:39:50,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-09T15:39:50,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:50,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-09T15:39:50,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-09T15:39:50,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=989ms, rpcTimeout=981ms 2024-11-09T15:39:50,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=989ms, rpcTimeout=981ms 2024-11-09T15:39:50,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:39:50,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=8182 (was 8259) 2024-11-09T15:39:50,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=8181 2024-11-09T15:39:50,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-09T15:39:50,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:50,317 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:50,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-09T15:39:50,318 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T15:39:50,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741845_1021 (size=416) 2024-11-09T15:39:50,731 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 25c9d1642c58775600c7f6d4076d8f1a, NAME => 'testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:50,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741846_1022 (size=67) 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 25c9d1642c58775600c7f6d4076d8f1a, disabling compactions & flushes 2024-11-09T15:39:50,740 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. after waiting 0 ms 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:50,740 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:50,740 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 25c9d1642c58775600c7f6d4076d8f1a: Waiting for close lock at 1731166790740Disabling compacts and flushes for region at 1731166790740Disabling writes for close at 1731166790740Writing region close event to WAL at 1731166790740Closed at 1731166790740 2024-11-09T15:39:50,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:50,743 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731166790743"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166790743"}]},"ts":"1731166790743"} 2024-11-09T15:39:50,746 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:50,747 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:50,747 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166790747"}]},"ts":"1731166790747"} 2024-11-09T15:39:50,750 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-09T15:39:50,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=25c9d1642c58775600c7f6d4076d8f1a, ASSIGN}] 2024-11-09T15:39:50,751 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=25c9d1642c58775600c7f6d4076d8f1a, ASSIGN 2024-11-09T15:39:50,753 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=25c9d1642c58775600c7f6d4076d8f1a, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-09T15:39:50,904 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=25c9d1642c58775600c7f6d4076d8f1a, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:50,907 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=25c9d1642c58775600c7f6d4076d8f1a, ASSIGN because future has completed 2024-11-09T15:39:50,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25c9d1642c58775600c7f6d4076d8f1a, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:51,067 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:51,067 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 25c9d1642c58775600c7f6d4076d8f1a, NAME => 'testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:51,068 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,068 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:51,068 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,068 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,070 INFO [StoreOpener-25c9d1642c58775600c7f6d4076d8f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,072 INFO [StoreOpener-25c9d1642c58775600c7f6d4076d8f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25c9d1642c58775600c7f6d4076d8f1a columnFamilyName testFamily 2024-11-09T15:39:51,072 DEBUG [StoreOpener-25c9d1642c58775600c7f6d4076d8f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:51,073 INFO [StoreOpener-25c9d1642c58775600c7f6d4076d8f1a-1 {}] regionserver.HStore(327): Store=25c9d1642c58775600c7f6d4076d8f1a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:51,073 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,074 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,075 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,075 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,075 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,079 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,089 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:51,090 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 25c9d1642c58775600c7f6d4076d8f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75185725, jitterRate=0.12035460770130157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:51,090 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:39:51,091 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 25c9d1642c58775600c7f6d4076d8f1a: Running coprocessor pre-open hook at 1731166791068Writing region info on filesystem at 1731166791068Initializing all the Stores at 1731166791070 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166791070Cleaning up temporary data from old regions at 1731166791075 (+5 ms)Running coprocessor post-open hooks at 1731166791090 (+15 ms)Region opened successfully at 1731166791091 (+1 ms) 2024-11-09T15:39:51,093 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a., pid=21, masterSystemTime=1731166791061 2024-11-09T15:39:51,097 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:51,097 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:39:51,098 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=25c9d1642c58775600c7f6d4076d8f1a, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:51,101 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25c9d1642c58775600c7f6d4076d8f1a, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:51,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-09T15:39:51,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 25c9d1642c58775600c7f6d4076d8f1a, server=da6b20b51b98,45207,1731166730521 in 195 msec 2024-11-09T15:39:51,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T15:39:51,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=25c9d1642c58775600c7f6d4076d8f1a, ASSIGN in 355 msec 2024-11-09T15:39:51,109 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:51,109 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166791109"}]},"ts":"1731166791109"} 2024-11-09T15:39:51,111 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-09T15:39:51,112 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:51,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 800 msec 2024-11-09T15:39:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T15:39:51,329 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-09T15:39:51,329 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-09T15:39:51,329 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:51,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-09T15:39:51,334 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:51,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-09T15:39:51,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:51,348 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:51,348 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-09T15:39:51,351 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T15:39:51,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T15:39:51,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T15:39:51,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-09T15:39:52,900 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-09T15:39:54,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455896' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T15:39:54,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46936 deadline: 1731166795363, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455896' does not exist 2024-11-09T15:39:54,364 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to da6b20b51b98,45207,1731166730521 for closing scanner id = -1567105645004455896 for 25c9d1642c58775600c7f6d4076d8f1a of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455896' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T15:39:54,369 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1700936484_22 at /127.0.0.1:60580 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=112 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=8077 (was 8181) 2024-11-09T15:39:54,378 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=112, ProcessCount=11, AvailableMemoryMB=8076 2024-11-09T15:39:54,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:39:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-09T15:39:54,383 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:39:54,383 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:54,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-09T15:39:54,385 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:39:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T15:39:54,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741847_1023 (size=411) 2024-11-09T15:39:54,796 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e93fedee689390fbe8e12e9f775aa53f, NAME => 'testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:39:54,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741848_1024 (size=62) 2024-11-09T15:39:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e93fedee689390fbe8e12e9f775aa53f, disabling compactions & flushes 2024-11-09T15:39:55,206 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. after waiting 0 ms 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,206 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,206 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e93fedee689390fbe8e12e9f775aa53f: Waiting for close lock at 1731166795206Disabling compacts and flushes for region at 1731166795206Disabling writes for close at 1731166795206Writing region close event to WAL at 1731166795206Closed at 1731166795206 2024-11-09T15:39:55,208 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:39:55,208 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731166795208"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166795208"}]},"ts":"1731166795208"} 2024-11-09T15:39:55,211 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:39:55,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:39:55,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166795212"}]},"ts":"1731166795212"} 2024-11-09T15:39:55,215 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-09T15:39:55,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e93fedee689390fbe8e12e9f775aa53f, ASSIGN}] 2024-11-09T15:39:55,217 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e93fedee689390fbe8e12e9f775aa53f, ASSIGN 2024-11-09T15:39:55,218 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e93fedee689390fbe8e12e9f775aa53f, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:39:55,369 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e93fedee689390fbe8e12e9f775aa53f, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:55,372 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e93fedee689390fbe8e12e9f775aa53f, ASSIGN because future has completed 2024-11-09T15:39:55,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e93fedee689390fbe8e12e9f775aa53f, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:39:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T15:39:55,531 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,531 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => e93fedee689390fbe8e12e9f775aa53f, NAME => 'testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:39:55,532 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,532 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:39:55,532 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,532 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,534 INFO [StoreOpener-e93fedee689390fbe8e12e9f775aa53f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,536 INFO [StoreOpener-e93fedee689390fbe8e12e9f775aa53f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e93fedee689390fbe8e12e9f775aa53f columnFamilyName testFamily 2024-11-09T15:39:55,536 DEBUG [StoreOpener-e93fedee689390fbe8e12e9f775aa53f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:39:55,537 INFO [StoreOpener-e93fedee689390fbe8e12e9f775aa53f-1 {}] regionserver.HStore(327): Store=e93fedee689390fbe8e12e9f775aa53f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:39:55,537 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,538 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,539 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,539 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,539 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,541 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,544 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:39:55,545 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened e93fedee689390fbe8e12e9f775aa53f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62259299, jitterRate=-0.07226414978504181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:39:55,545 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:39:55,545 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for e93fedee689390fbe8e12e9f775aa53f: Running coprocessor pre-open hook at 1731166795532Writing region info on filesystem at 1731166795532Initializing all the Stores at 1731166795533 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166795533Cleaning up temporary data from old regions at 1731166795539 (+6 ms)Running coprocessor post-open hooks at 1731166795545 (+6 ms)Region opened successfully at 1731166795545 2024-11-09T15:39:55,546 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f., pid=24, masterSystemTime=1731166795525 2024-11-09T15:39:55,549 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,549 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:39:55,550 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e93fedee689390fbe8e12e9f775aa53f, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:39:55,552 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e93fedee689390fbe8e12e9f775aa53f, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:39:55,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T15:39:55,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e93fedee689390fbe8e12e9f775aa53f, server=da6b20b51b98,45207,1731166730521 in 181 msec 2024-11-09T15:39:55,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T15:39:55,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=e93fedee689390fbe8e12e9f775aa53f, ASSIGN in 341 msec 2024-11-09T15:39:55,561 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:39:55,561 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166795561"}]},"ts":"1731166795561"} 2024-11-09T15:39:55,564 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-09T15:39:55,565 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:39:55,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1850 sec 2024-11-09T15:39:56,570 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:39:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T15:39:57,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-09T15:39:57,419 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-09T15:39:57,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:57,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-09T15:39:57,423 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:39:57,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-09T15:39:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:39:57,438 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:39:57,438 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-09T15:39:57,441 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T15:39:57,443 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T15:39:57,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T15:39:57,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-09T15:39:58,901 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-09T15:40:00,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-09T15:40:00,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-09T15:40:00,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-09T15:40:00,249 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T15:40:00,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455892' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T15:40:00,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46936 deadline: 1731166801453, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455892' does not exist 2024-11-09T15:40:00,454 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to da6b20b51b98,45207,1731166730521 for closing scanner id = -1567105645004455892 for e93fedee689390fbe8e12e9f775aa53f of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455892' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T15:40:00,460 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 112), ProcessCount=11 (was 11), AvailableMemoryMB=7982 (was 8076) 2024-11-09T15:40:00,469 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=7981 2024-11-09T15:40:00,471 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:40:00,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:00,474 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:40:00,474 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:00,474 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-09T15:40:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T15:40:00,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:40:00,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741849_1025 (size=419) 2024-11-09T15:40:00,887 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4eef2c1f334291350279700770917d45, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:40:00,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741850_1026 (size=70) 2024-11-09T15:40:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4eef2c1f334291350279700770917d45, disabling compactions & flushes 2024-11-09T15:40:01,297 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. after waiting 0 ms 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,297 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,297 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4eef2c1f334291350279700770917d45: Waiting for close lock at 1731166801297Disabling compacts and flushes for region at 1731166801297Disabling writes for close at 1731166801297Writing region close event to WAL at 1731166801297Closed at 1731166801297 2024-11-09T15:40:01,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:40:01,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731166801299"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166801299"}]},"ts":"1731166801299"} 2024-11-09T15:40:01,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:40:01,304 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:40:01,304 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166801304"}]},"ts":"1731166801304"} 2024-11-09T15:40:01,306 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-09T15:40:01,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=4eef2c1f334291350279700770917d45, ASSIGN}] 2024-11-09T15:40:01,308 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=4eef2c1f334291350279700770917d45, ASSIGN 2024-11-09T15:40:01,309 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=4eef2c1f334291350279700770917d45, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:40:01,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4eef2c1f334291350279700770917d45, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:01,463 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=4eef2c1f334291350279700770917d45, ASSIGN because future has completed 2024-11-09T15:40:01,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eef2c1f334291350279700770917d45, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:40:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T15:40:01,622 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 4eef2c1f334291350279700770917d45, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:40:01,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:01,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,625 INFO [StoreOpener-4eef2c1f334291350279700770917d45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,627 INFO [StoreOpener-4eef2c1f334291350279700770917d45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4eef2c1f334291350279700770917d45 columnFamilyName testFamily 2024-11-09T15:40:01,627 DEBUG [StoreOpener-4eef2c1f334291350279700770917d45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:01,627 INFO [StoreOpener-4eef2c1f334291350279700770917d45-1 {}] regionserver.HStore(327): Store=4eef2c1f334291350279700770917d45/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:40:01,628 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,629 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,629 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,632 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,636 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:40:01,637 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 4eef2c1f334291350279700770917d45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67076932, jitterRate=-4.7582387924194336E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:40:01,638 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:01,638 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 4eef2c1f334291350279700770917d45: Running coprocessor pre-open hook at 1731166801623Writing region info on filesystem at 1731166801623Initializing all the Stores at 1731166801624 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166801624Cleaning up temporary data from old regions at 1731166801630 (+6 ms)Running coprocessor post-open hooks at 1731166801638 (+8 ms)Region opened successfully at 1731166801638 2024-11-09T15:40:01,640 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., pid=27, masterSystemTime=1731166801616 2024-11-09T15:40:01,643 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,643 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:01,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=4eef2c1f334291350279700770917d45, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:01,647 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4eef2c1f334291350279700770917d45, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:40:01,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T15:40:01,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 4eef2c1f334291350279700770917d45, server=da6b20b51b98,45207,1731166730521 in 185 msec 2024-11-09T15:40:01,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T15:40:01,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=4eef2c1f334291350279700770917d45, ASSIGN in 345 msec 2024-11-09T15:40:01,654 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:40:01,655 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166801654"}]},"ts":"1731166801654"} 2024-11-09T15:40:01,657 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-09T15:40:01,658 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:40:01,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1870 sec 2024-11-09T15:40:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T15:40:03,509 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-09T15:40:03,509 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-09T15:40:03,509 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:03,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-09T15:40:03,515 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:03,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-09T15:40:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:40:03,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:40:03,529 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-09T15:40:03,532 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T15:40:03,534 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-09T15:40:03,561 INFO [master/da6b20b51b98:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T15:40:03,561 INFO [master/da6b20b51b98:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T15:40:04,903 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-09T15:40:07,147 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:40:09,540 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-09T15:40:09,540 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-09T15:40:09,540 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:40:09,550 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=87 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=7872 (was 7981) 2024-11-09T15:40:09,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=87, ProcessCount=11, AvailableMemoryMB=7872 2024-11-09T15:40:09,561 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:40:09,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-09T15:40:09,564 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:40:09,564 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:09,564 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-09T15:40:09,565 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:40:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T15:40:09,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741851_1027 (size=407) 2024-11-09T15:40:09,976 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 987459392592f1ceaee30b4a5bd3e7f0, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:40:09,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741852_1028 (size=58) 2024-11-09T15:40:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-09T15:40:10,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:10,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T15:40:10,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:10,386 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 987459392592f1ceaee30b4a5bd3e7f0, disabling compactions & flushes 2024-11-09T15:40:10,386 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,387 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,387 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. after waiting 0 ms 2024-11-09T15:40:10,387 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,387 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,387 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 987459392592f1ceaee30b4a5bd3e7f0: Waiting for close lock at 1731166810386Disabling compacts and flushes for region at 1731166810386Disabling writes for close at 1731166810387 (+1 ms)Writing region close event to WAL at 1731166810387Closed at 1731166810387 2024-11-09T15:40:10,388 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:40:10,389 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731166810389"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166810389"}]},"ts":"1731166810389"} 2024-11-09T15:40:10,391 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:40:10,393 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:40:10,393 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166810393"}]},"ts":"1731166810393"} 2024-11-09T15:40:10,395 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-09T15:40:10,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=987459392592f1ceaee30b4a5bd3e7f0, ASSIGN}] 2024-11-09T15:40:10,396 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=987459392592f1ceaee30b4a5bd3e7f0, ASSIGN 2024-11-09T15:40:10,397 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=987459392592f1ceaee30b4a5bd3e7f0, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:40:10,548 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=987459392592f1ceaee30b4a5bd3e7f0, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:10,551 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=987459392592f1ceaee30b4a5bd3e7f0, ASSIGN because future has completed 2024-11-09T15:40:10,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 987459392592f1ceaee30b4a5bd3e7f0, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:40:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T15:40:10,710 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 987459392592f1ceaee30b4a5bd3e7f0, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:40:10,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:10,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,713 INFO [StoreOpener-987459392592f1ceaee30b4a5bd3e7f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,715 INFO [StoreOpener-987459392592f1ceaee30b4a5bd3e7f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 987459392592f1ceaee30b4a5bd3e7f0 columnFamilyName testFamily 2024-11-09T15:40:10,715 DEBUG [StoreOpener-987459392592f1ceaee30b4a5bd3e7f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:10,716 INFO [StoreOpener-987459392592f1ceaee30b4a5bd3e7f0-1 {}] regionserver.HStore(327): Store=987459392592f1ceaee30b4a5bd3e7f0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:40:10,716 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,718 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,718 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,719 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:40:10,722 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 987459392592f1ceaee30b4a5bd3e7f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72373338, jitterRate=0.07844677567481995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:40:10,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:10,723 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 987459392592f1ceaee30b4a5bd3e7f0: Running coprocessor pre-open hook at 1731166810711Writing region info on filesystem at 1731166810711Initializing all the Stores at 1731166810712 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166810712Cleaning up temporary data from old regions at 1731166810718 (+6 ms)Running coprocessor post-open hooks at 1731166810723 (+5 ms)Region opened successfully at 1731166810723 2024-11-09T15:40:10,724 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., pid=30, masterSystemTime=1731166810704 2024-11-09T15:40:10,727 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,727 INFO [RS_OPEN_PRIORITY_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:10,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=987459392592f1ceaee30b4a5bd3e7f0, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:10,730 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 987459392592f1ceaee30b4a5bd3e7f0, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:40:10,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-09T15:40:10,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 987459392592f1ceaee30b4a5bd3e7f0, server=da6b20b51b98,45207,1731166730521 in 179 msec 2024-11-09T15:40:10,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T15:40:10,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=987459392592f1ceaee30b4a5bd3e7f0, ASSIGN in 338 msec 2024-11-09T15:40:10,737 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:40:10,737 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166810737"}]},"ts":"1731166810737"} 2024-11-09T15:40:10,739 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-09T15:40:10,740 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:40:10,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1790 sec 2024-11-09T15:40:10,905 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-09T15:40:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T15:40:12,598 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-09T15:40:12,598 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-09T15:40:12,598 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:12,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-09T15:40:12,604 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:12,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-09T15:40:12,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:40:12,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:40:12,617 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-09T15:40:12,619 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T15:40:12,621 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T15:40:12,622 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T15:40:12,622 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-09T15:40:16,212 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:40:18,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455884' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T15:40:18,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45207 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:46936 deadline: 1731166824631, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455884' does not exist 2024-11-09T15:40:18,635 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to da6b20b51b98,45207,1731166730521 for closing scanner id = -1567105645004455884 for 987459392592f1ceaee30b4a5bd3e7f0 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-1567105645004455884' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T15:40:18,642 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=90 (was 87) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7879 (was 7872) - AvailableMemoryMB LEAK? - 2024-11-09T15:40:18,651 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=90, ProcessCount=11, AvailableMemoryMB=7879 2024-11-09T15:40:18,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T15:40:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:18,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T15:40:18,656 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:18,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-09T15:40:18,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T15:40:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T15:40:18,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741853_1029 (size=423) 2024-11-09T15:40:18,929 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T15:40:19,068 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6dfbe7a22c10fc6b492b83dc9c302881, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a 2024-11-09T15:40:19,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741854_1030 (size=74) 2024-11-09T15:40:19,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-09T15:40:19,477 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:19,477 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6dfbe7a22c10fc6b492b83dc9c302881, disabling compactions & flushes 2024-11-09T15:40:19,477 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,477 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,477 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. after waiting 0 ms 2024-11-09T15:40:19,477 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,477 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,478 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6dfbe7a22c10fc6b492b83dc9c302881: Waiting for close lock at 1731166819477Disabling compacts and flushes for region at 1731166819477Disabling writes for close at 1731166819477Writing region close event to WAL at 1731166819477Closed at 1731166819477 2024-11-09T15:40:19,479 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T15:40:19,480 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731166819479"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731166819479"}]},"ts":"1731166819479"} 2024-11-09T15:40:19,482 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T15:40:19,484 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T15:40:19,484 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166819484"}]},"ts":"1731166819484"} 2024-11-09T15:40:19,486 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-09T15:40:19,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6dfbe7a22c10fc6b492b83dc9c302881, ASSIGN}] 2024-11-09T15:40:19,488 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6dfbe7a22c10fc6b492b83dc9c302881, ASSIGN 2024-11-09T15:40:19,489 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6dfbe7a22c10fc6b492b83dc9c302881, ASSIGN; state=OFFLINE, location=da6b20b51b98,45207,1731166730521; forceNewPlan=false, retain=false 2024-11-09T15:40:19,639 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6dfbe7a22c10fc6b492b83dc9c302881, regionState=OPENING, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:19,643 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6dfbe7a22c10fc6b492b83dc9c302881, ASSIGN because future has completed 2024-11-09T15:40:19,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dfbe7a22c10fc6b492b83dc9c302881, server=da6b20b51b98,45207,1731166730521}] 2024-11-09T15:40:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T15:40:19,801 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,802 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dfbe7a22c10fc6b492b83dc9c302881, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881.', STARTKEY => '', ENDKEY => ''} 2024-11-09T15:40:19,802 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,802 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T15:40:19,802 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,802 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,804 INFO [StoreOpener-6dfbe7a22c10fc6b492b83dc9c302881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,806 INFO [StoreOpener-6dfbe7a22c10fc6b492b83dc9c302881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dfbe7a22c10fc6b492b83dc9c302881 columnFamilyName testFamily 2024-11-09T15:40:19,806 DEBUG [StoreOpener-6dfbe7a22c10fc6b492b83dc9c302881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T15:40:19,807 INFO [StoreOpener-6dfbe7a22c10fc6b492b83dc9c302881-1 {}] regionserver.HStore(327): Store=6dfbe7a22c10fc6b492b83dc9c302881/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T15:40:19,807 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,808 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,809 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,809 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,809 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,811 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,814 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T15:40:19,814 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 6dfbe7a22c10fc6b492b83dc9c302881; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64689713, jitterRate=-0.03604815900325775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T15:40:19,814 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:19,815 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 6dfbe7a22c10fc6b492b83dc9c302881: Running coprocessor pre-open hook at 1731166819802Writing region info on filesystem at 1731166819802Initializing all the Stores at 1731166819804 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731166819804Cleaning up temporary data from old regions at 1731166819809 (+5 ms)Running coprocessor post-open hooks at 1731166819814 (+5 ms)Region opened successfully at 1731166819815 (+1 ms) 2024-11-09T15:40:19,816 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., pid=33, masterSystemTime=1731166819796 2024-11-09T15:40:19,819 DEBUG [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,819 INFO [RS_OPEN_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:19,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6dfbe7a22c10fc6b492b83dc9c302881, regionState=OPEN, openSeqNum=2, regionLocation=da6b20b51b98,45207,1731166730521 2024-11-09T15:40:19,822 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dfbe7a22c10fc6b492b83dc9c302881, server=da6b20b51b98,45207,1731166730521 because future has completed 2024-11-09T15:40:19,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-09T15:40:19,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dfbe7a22c10fc6b492b83dc9c302881, server=da6b20b51b98,45207,1731166730521 in 180 msec 2024-11-09T15:40:19,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T15:40:19,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6dfbe7a22c10fc6b492b83dc9c302881, ASSIGN in 340 msec 2024-11-09T15:40:19,829 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T15:40:19,830 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731166819829"}]},"ts":"1731166819829"} 2024-11-09T15:40:19,832 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-09T15:40:19,833 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T15:40:19,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1800 sec 2024-11-09T15:40:20,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:20,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T15:40:20,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-09T15:40:20,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-09T15:40:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T15:40:21,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-09T15:40:21,679 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-09T15:40:21,679 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:21,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-09T15:40:21,684 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T15:40:21,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-09T15:40:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:40:21,698 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T15:40:21,698 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-09T15:40:21,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T15:40:21,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T15:40:22,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-09T15:40:22,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-09T15:40:22,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:40:22,906 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-09T15:40:23,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-09T15:40:23,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2] 2024-11-09T15:40:23,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45207 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T15:40:24,559 DEBUG [master/da6b20b51b98:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T15:40:24,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=da6b20b51b98:45207 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=988ms, rpcTimeout=980ms 2024-11-09T15:40:24,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=988ms, rpcTimeout=980ms 2024-11-09T15:40:24,710 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., hostname=da6b20b51b98,45207,1731166730521, seqNum=2 from cache 2024-11-09T15:40:24,719 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=91 (was 90) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7864 (was 7879) 2024-11-09T15:40:24,720 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T15:40:24,720 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T15:40:24,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T15:40:24,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T15:40:24,726 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T15:40:24,726 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T15:40:24,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T15:40:24,727 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T15:40:24,727 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=721756251, stopped=false 2024-11-09T15:40:24,727 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da6b20b51b98,34975,1731166729839 2024-11-09T15:40:24,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T15:40:24,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T15:40:24,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:40:24,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:40:24,729 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T15:40:24,730 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T15:40:24,730 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T15:40:24,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T15:40:24,730 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T15:40:24,731 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da6b20b51b98,45207,1731166730521' ***** 2024-11-09T15:40:24,731 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T15:40:24,731 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T15:40:24,731 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T15:40:24,731 INFO [RS:0;da6b20b51b98:45207 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T15:40:24,731 INFO [RS:0;da6b20b51b98:45207 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 25c9d1642c58775600c7f6d4076d8f1a 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 4eef2c1f334291350279700770917d45 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dfbe7a22c10fc6b492b83dc9c302881 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 55ba359dcd9c97c003d064a2efb76221 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 8812061f09cf80dc49527a5a0a9f5a00 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 45ba5a0d25f5f40ef5c8117cd0fa3177 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for ab830ca42595810345ca62bc8a3c2219 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 987459392592f1ceaee30b4a5bd3e7f0 2024-11-09T15:40:24,732 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(959): stopping server da6b20b51b98,45207,1731166730521 2024-11-09T15:40:24,733 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25c9d1642c58775600c7f6d4076d8f1a, disabling compactions & flushes 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T15:40:24,733 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:40:24,733 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da6b20b51b98:45207. 2024-11-09T15:40:24,733 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. after waiting 0 ms 2024-11-09T15:40:24,733 DEBUG [RS:0;da6b20b51b98:45207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T15:40:24,733 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:40:24,733 DEBUG [RS:0;da6b20b51b98:45207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T15:40:24,733 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T15:40:24,734 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-09T15:40:24,734 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T15:40:24,734 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T15:40:24,734 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T15:40:24,734 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T15:40:24,734 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 25c9d1642c58775600c7f6d4076d8f1a, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:24,734 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T15:40:24,736 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-09T15:40:24,736 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 25c9d1642c58775600c7f6d4076d8f1a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:24,794 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/.tmp/testFamily/4960a36305aa4d008c37f05579770b28 is 61, key is row-0/testFamily:testQualifier/1731166791339/Put/seqid=0 2024-11-09T15:40:24,797 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/info/afb4caaf0a2149e8b2f03f0f358ed072 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177./info:regioninfo/1731166765015/Put/seqid=0 2024-11-09T15:40:24,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741856_1032 (size=5049) 2024-11-09T15:40:24,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741855_1031 (size=20564) 2024-11-09T15:40:24,913 INFO [regionserver/da6b20b51b98:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T15:40:24,914 INFO [regionserver/da6b20b51b98:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T15:40:24,934 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 25c9d1642c58775600c7f6d4076d8f1a, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:25,135 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 25c9d1642c58775600c7f6d4076d8f1a, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:25,207 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/.tmp/testFamily/4960a36305aa4d008c37f05579770b28 2024-11-09T15:40:25,207 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/info/afb4caaf0a2149e8b2f03f0f358ed072 2024-11-09T15:40:25,223 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T15:40:25,253 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/.tmp/testFamily/4960a36305aa4d008c37f05579770b28 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/testFamily/4960a36305aa4d008c37f05579770b28 2024-11-09T15:40:25,263 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/testFamily/4960a36305aa4d008c37f05579770b28, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:25,272 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/ns/9cc1413a0f0f497b8cb0ce48702e2082 is 43, key is default/ns:d/1731166763459/Put/seqid=0 2024-11-09T15:40:25,272 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 25c9d1642c58775600c7f6d4076d8f1a in 535ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:25,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741857_1033 (size=5153) 2024-11-09T15:40:25,279 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNextAsync/25c9d1642c58775600c7f6d4076d8f1a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:25,282 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:40:25,282 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25c9d1642c58775600c7f6d4076d8f1a: Waiting for close lock at 1731166824732Running coprocessor pre-close hooks at 1731166824732Disabling compacts and flushes for region at 1731166824733 (+1 ms)Disabling writes for close at 1731166824733Obtaining lock to block concurrent updates at 1731166824736 (+3 ms)Preparing flush snapshotting stores in 25c9d1642c58775600c7f6d4076d8f1a at 1731166824736Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166824744 (+8 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. at 1731166824746 (+2 ms)Flushing 25c9d1642c58775600c7f6d4076d8f1a/testFamily: creating writer at 1731166824747 (+1 ms)Flushing 25c9d1642c58775600c7f6d4076d8f1a/testFamily: appending metadata at 1731166824789 (+42 ms)Flushing 25c9d1642c58775600c7f6d4076d8f1a/testFamily: closing flushed file at 1731166824791 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1523e1: reopening flushed file at 1731166825251 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 25c9d1642c58775600c7f6d4076d8f1a in 535ms, sequenceid=8, compaction requested=false at 1731166825272 (+21 ms)Writing region close event to WAL at 1731166825274 (+2 ms)Running coprocessor post-close hooks at 1731166825280 (+6 ms)Closed at 1731166825282 (+2 ms) 2024-11-09T15:40:25,283 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731166790311.25c9d1642c58775600c7f6d4076d8f1a. 2024-11-09T15:40:25,283 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4eef2c1f334291350279700770917d45, disabling compactions & flushes 2024-11-09T15:40:25,283 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:25,283 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:25,283 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. after waiting 0 ms 2024-11-09T15:40:25,283 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:25,283 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4eef2c1f334291350279700770917d45 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:25,301 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/.tmp/testFamily/11a14b6cecc74704bc71b39f27e7067e is 61, key is row-0/testFamily:testQualifier/1731166803520/Put/seqid=0 2024-11-09T15:40:25,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:40:25,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741858_1034 (size=5049) 2024-11-09T15:40:25,335 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:25,535 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:25,678 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/ns/9cc1413a0f0f497b8cb0ce48702e2082 2024-11-09T15:40:25,705 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/table/684bf5b4860f4fe68590d72fb207d12a is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731166765031/Put/seqid=0 2024-11-09T15:40:25,709 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/.tmp/testFamily/11a14b6cecc74704bc71b39f27e7067e 2024-11-09T15:40:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741859_1035 (size=6699) 2024-11-09T15:40:25,712 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/table/684bf5b4860f4fe68590d72fb207d12a 2024-11-09T15:40:25,719 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/.tmp/testFamily/11a14b6cecc74704bc71b39f27e7067e as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/testFamily/11a14b6cecc74704bc71b39f27e7067e 2024-11-09T15:40:25,721 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/info/afb4caaf0a2149e8b2f03f0f358ed072 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/info/afb4caaf0a2149e8b2f03f0f358ed072 2024-11-09T15:40:25,728 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/testFamily/11a14b6cecc74704bc71b39f27e7067e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:25,729 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/info/afb4caaf0a2149e8b2f03f0f358ed072, entries=100, sequenceid=56, filesize=20.1 K 2024-11-09T15:40:25,729 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4eef2c1f334291350279700770917d45 in 446ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:25,731 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/ns/9cc1413a0f0f497b8cb0ce48702e2082 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/ns/9cc1413a0f0f497b8cb0ce48702e2082 2024-11-09T15:40:25,735 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/4eef2c1f334291350279700770917d45/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:25,735 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-09T15:40:25,736 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 45ba5a0d25f5f40ef5c8117cd0fa3177, 4eef2c1f334291350279700770917d45, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:25,736 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:25,736 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4eef2c1f334291350279700770917d45: Waiting for close lock at 1731166825283Running coprocessor pre-close hooks at 1731166825283Disabling compacts and flushes for region at 1731166825283Disabling writes for close at 1731166825283Obtaining lock to block concurrent updates at 1731166825283Preparing flush snapshotting stores in 4eef2c1f334291350279700770917d45 at 1731166825283Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166825284 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. at 1731166825285 (+1 ms)Flushing 4eef2c1f334291350279700770917d45/testFamily: creating writer at 1731166825285Flushing 4eef2c1f334291350279700770917d45/testFamily: appending metadata at 1731166825301 (+16 ms)Flushing 4eef2c1f334291350279700770917d45/testFamily: closing flushed file at 1731166825301Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@643e83c3: reopening flushed file at 1731166825718 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4eef2c1f334291350279700770917d45 in 446ms, sequenceid=8, compaction requested=false at 1731166825729 (+11 ms)Writing region close event to WAL at 1731166825730 (+1 ms)Running coprocessor post-close hooks at 1731166825736 (+6 ms)Closed at 1731166825736 2024-11-09T15:40:25,736 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731166800471.4eef2c1f334291350279700770917d45. 2024-11-09T15:40:25,736 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dfbe7a22c10fc6b492b83dc9c302881, disabling compactions & flushes 2024-11-09T15:40:25,737 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:25,737 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:25,737 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. after waiting 0 ms 2024-11-09T15:40:25,737 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:25,737 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6dfbe7a22c10fc6b492b83dc9c302881 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:25,739 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/ns/9cc1413a0f0f497b8cb0ce48702e2082, entries=2, sequenceid=56, filesize=5.0 K 2024-11-09T15:40:25,741 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/.tmp/table/684bf5b4860f4fe68590d72fb207d12a as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/table/684bf5b4860f4fe68590d72fb207d12a 2024-11-09T15:40:25,750 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/table/684bf5b4860f4fe68590d72fb207d12a, entries=20, sequenceid=56, filesize=6.5 K 2024-11-09T15:40:25,751 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1017ms, sequenceid=56, compaction requested=false 2024-11-09T15:40:25,756 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/.tmp/testFamily/3dcc4655de9c4329a490e4c8e297a618 is 61, key is row-0/testFamily:testQualifier/1731166821689/Put/seqid=0 2024-11-09T15:40:25,758 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-09T15:40:25,759 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T15:40:25,759 INFO [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T15:40:25,759 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731166824734Running coprocessor pre-close hooks at 1731166824734Disabling compacts and flushes for region at 1731166824734Disabling writes for close at 1731166824734Obtaining lock to block concurrent updates at 1731166824736 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731166824736Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731166824744 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731166824746 (+2 ms)Flushing 1588230740/info: creating writer at 1731166824747 (+1 ms)Flushing 1588230740/info: appending metadata at 1731166824794 (+47 ms)Flushing 1588230740/info: closing flushed file at 1731166824794Flushing 1588230740/ns: creating writer at 1731166825251 (+457 ms)Flushing 1588230740/ns: appending metadata at 1731166825271 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731166825271Flushing 1588230740/table: creating writer at 1731166825687 (+416 ms)Flushing 1588230740/table: appending metadata at 1731166825705 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731166825705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35efe0: reopening flushed file at 1731166825720 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dea24d: reopening flushed file at 1731166825729 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f1907d8: reopening flushed file at 1731166825740 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1017ms, sequenceid=56, compaction requested=false at 1731166825751 (+11 ms)Writing region close event to WAL at 1731166825753 (+2 ms)Running coprocessor post-close hooks at 1731166825758 (+5 ms)Closed at 1731166825759 (+1 ms) 2024-11-09T15:40:25,759 DEBUG [RS_CLOSE_META-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T15:40:25,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741860_1036 (size=5049) 2024-11-09T15:40:25,936 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:26,136 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 6dfbe7a22c10fc6b492b83dc9c302881, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:26,163 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/.tmp/testFamily/3dcc4655de9c4329a490e4c8e297a618 2024-11-09T15:40:26,173 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/.tmp/testFamily/3dcc4655de9c4329a490e4c8e297a618 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/testFamily/3dcc4655de9c4329a490e4c8e297a618 2024-11-09T15:40:26,180 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/testFamily/3dcc4655de9c4329a490e4c8e297a618, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:26,181 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6dfbe7a22c10fc6b492b83dc9c302881 in 444ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:26,186 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScannerAsync/6dfbe7a22c10fc6b492b83dc9c302881/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:26,187 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:26,187 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dfbe7a22c10fc6b492b83dc9c302881: Waiting for close lock at 1731166825736Running coprocessor pre-close hooks at 1731166825736Disabling compacts and flushes for region at 1731166825736Disabling writes for close at 1731166825737 (+1 ms)Obtaining lock to block concurrent updates at 1731166825737Preparing flush snapshotting stores in 6dfbe7a22c10fc6b492b83dc9c302881 at 1731166825737Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166825737Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. at 1731166825738 (+1 ms)Flushing 6dfbe7a22c10fc6b492b83dc9c302881/testFamily: creating writer at 1731166825738Flushing 6dfbe7a22c10fc6b492b83dc9c302881/testFamily: appending metadata at 1731166825756 (+18 ms)Flushing 6dfbe7a22c10fc6b492b83dc9c302881/testFamily: closing flushed file at 1731166825756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fc402be: reopening flushed file at 1731166826171 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6dfbe7a22c10fc6b492b83dc9c302881 in 444ms, sequenceid=8, compaction requested=false at 1731166826181 (+10 ms)Writing region close event to WAL at 1731166826182 (+1 ms)Running coprocessor post-close hooks at 1731166826187 (+5 ms)Closed at 1731166826187 2024-11-09T15:40:26,187 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731166818653.6dfbe7a22c10fc6b492b83dc9c302881. 2024-11-09T15:40:26,187 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e93fedee689390fbe8e12e9f775aa53f, disabling compactions & flushes 2024-11-09T15:40:26,187 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:40:26,187 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:40:26,187 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. after waiting 0 ms 2024-11-09T15:40:26,188 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:40:26,188 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e93fedee689390fbe8e12e9f775aa53f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:26,205 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/.tmp/testFamily/644d758e057b45c590be6a7b30871cca is 61, key is row-0/testFamily:testQualifier/1731166797428/Put/seqid=0 2024-11-09T15:40:26,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741861_1037 (size=5049) 2024-11-09T15:40:26,336 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:26,537 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192, e93fedee689390fbe8e12e9f775aa53f 2024-11-09T15:40:26,611 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/.tmp/testFamily/644d758e057b45c590be6a7b30871cca 2024-11-09T15:40:26,621 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/.tmp/testFamily/644d758e057b45c590be6a7b30871cca as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/testFamily/644d758e057b45c590be6a7b30871cca 2024-11-09T15:40:26,629 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/testFamily/644d758e057b45c590be6a7b30871cca, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:26,630 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e93fedee689390fbe8e12e9f775aa53f in 442ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:26,635 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnNext/e93fedee689390fbe8e12e9f775aa53f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:26,635 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:40:26,635 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e93fedee689390fbe8e12e9f775aa53f: Waiting for close lock at 1731166826187Running coprocessor pre-close hooks at 1731166826187Disabling compacts and flushes for region at 1731166826187Disabling writes for close at 1731166826187Obtaining lock to block concurrent updates at 1731166826188 (+1 ms)Preparing flush snapshotting stores in e93fedee689390fbe8e12e9f775aa53f at 1731166826188Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166826188Flushing stores of testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. at 1731166826188Flushing e93fedee689390fbe8e12e9f775aa53f/testFamily: creating writer at 1731166826188Flushing e93fedee689390fbe8e12e9f775aa53f/testFamily: appending metadata at 1731166826205 (+17 ms)Flushing e93fedee689390fbe8e12e9f775aa53f/testFamily: closing flushed file at 1731166826205Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51fc478f: reopening flushed file at 1731166826620 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e93fedee689390fbe8e12e9f775aa53f in 442ms, sequenceid=8, compaction requested=false at 1731166826630 (+10 ms)Writing region close event to WAL at 1731166826631 (+1 ms)Running coprocessor post-close hooks at 1731166826635 (+4 ms)Closed at 1731166826635 2024-11-09T15:40:26,636 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731166794380.e93fedee689390fbe8e12e9f775aa53f. 2024-11-09T15:40:26,636 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55ba359dcd9c97c003d064a2efb76221, disabling compactions & flushes 2024-11-09T15:40:26,636 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:40:26,636 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:40:26,636 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. after waiting 0 ms 2024-11-09T15:40:26,636 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:40:26,636 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 55ba359dcd9c97c003d064a2efb76221 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:26,653 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/.tmp/testFamily/84c4e38cb8e8437489c34d7ab176ab3d is 61, key is row-0/testFamily:testQualifier/1731166777102/Put/seqid=0 2024-11-09T15:40:26,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741862_1038 (size=5049) 2024-11-09T15:40:26,737 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-09T15:40:26,737 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1325): Online Regions={55ba359dcd9c97c003d064a2efb76221=hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221., 8812061f09cf80dc49527a5a0a9f5a00=testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., 45ba5a0d25f5f40ef5c8117cd0fa3177=testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177., ab830ca42595810345ca62bc8a3c2219=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., 987459392592f1ceaee30b4a5bd3e7f0=hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., bd1cf2ea3c38eda30f839dfd5f4eb192=testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.} 2024-11-09T15:40:26,737 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:26,938 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 55ba359dcd9c97c003d064a2efb76221, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,059 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/.tmp/testFamily/84c4e38cb8e8437489c34d7ab176ab3d 2024-11-09T15:40:27,067 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/.tmp/testFamily/84c4e38cb8e8437489c34d7ab176ab3d as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/testFamily/84c4e38cb8e8437489c34d7ab176ab3d 2024-11-09T15:40:27,074 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/testFamily/84c4e38cb8e8437489c34d7ab176ab3d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:27,075 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55ba359dcd9c97c003d064a2efb76221 in 439ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:27,080 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNextAsync/55ba359dcd9c97c003d064a2efb76221/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:27,081 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55ba359dcd9c97c003d064a2efb76221: Waiting for close lock at 1731166826636Running coprocessor pre-close hooks at 1731166826636Disabling compacts and flushes for region at 1731166826636Disabling writes for close at 1731166826636Obtaining lock to block concurrent updates at 1731166826636Preparing flush snapshotting stores in 55ba359dcd9c97c003d064a2efb76221 at 1731166826636Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166826636Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. at 1731166826637 (+1 ms)Flushing 55ba359dcd9c97c003d064a2efb76221/testFamily: creating writer at 1731166826637Flushing 55ba359dcd9c97c003d064a2efb76221/testFamily: appending metadata at 1731166826653 (+16 ms)Flushing 55ba359dcd9c97c003d064a2efb76221/testFamily: closing flushed file at 1731166826653Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3088a298: reopening flushed file at 1731166827066 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 55ba359dcd9c97c003d064a2efb76221 in 439ms, sequenceid=8, compaction requested=false at 1731166827075 (+9 ms)Writing region close event to WAL at 1731166827076 (+1 ms)Running coprocessor post-close hooks at 1731166827080 (+4 ms)Closed at 1731166827080 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731166774055.55ba359dcd9c97c003d064a2efb76221. 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8812061f09cf80dc49527a5a0a9f5a00, disabling compactions & flushes 2024-11-09T15:40:27,081 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. after waiting 0 ms 2024-11-09T15:40:27,081 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:40:27,081 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8812061f09cf80dc49527a5a0a9f5a00 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:27,097 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/.tmp/testFamily/2c9876b45a574a15a3fa6014bc10c84d is 61, key is row-0/testFamily:testQualifier/1731166787270/Put/seqid=0 2024-11-09T15:40:27,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741863_1039 (size=5049) 2024-11-09T15:40:27,138 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,338 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 8812061f09cf80dc49527a5a0a9f5a00, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,503 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/.tmp/testFamily/2c9876b45a574a15a3fa6014bc10c84d 2024-11-09T15:40:27,511 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/.tmp/testFamily/2c9876b45a574a15a3fa6014bc10c84d as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/testFamily/2c9876b45a574a15a3fa6014bc10c84d 2024-11-09T15:40:27,518 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/testFamily/2c9876b45a574a15a3fa6014bc10c84d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:27,520 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8812061f09cf80dc49527a5a0a9f5a00 in 438ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:27,524 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testNormalScanTimeoutOnOpenScanner/8812061f09cf80dc49527a5a0a9f5a00/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:27,524 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8812061f09cf80dc49527a5a0a9f5a00: Waiting for close lock at 1731166827081Running coprocessor pre-close hooks at 1731166827081Disabling compacts and flushes for region at 1731166827081Disabling writes for close at 1731166827081Obtaining lock to block concurrent updates at 1731166827081Preparing flush snapshotting stores in 8812061f09cf80dc49527a5a0a9f5a00 at 1731166827081Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166827081Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. at 1731166827082 (+1 ms)Flushing 8812061f09cf80dc49527a5a0a9f5a00/testFamily: creating writer at 1731166827082Flushing 8812061f09cf80dc49527a5a0a9f5a00/testFamily: appending metadata at 1731166827097 (+15 ms)Flushing 8812061f09cf80dc49527a5a0a9f5a00/testFamily: closing flushed file at 1731166827097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ea84b60: reopening flushed file at 1731166827510 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8812061f09cf80dc49527a5a0a9f5a00 in 438ms, sequenceid=8, compaction requested=false at 1731166827520 (+10 ms)Writing region close event to WAL at 1731166827520Running coprocessor post-close hooks at 1731166827524 (+4 ms)Closed at 1731166827524 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731166784231.8812061f09cf80dc49527a5a0a9f5a00. 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45ba5a0d25f5f40ef5c8117cd0fa3177, disabling compactions & flushes 2024-11-09T15:40:27,525 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. after waiting 0 ms 2024-11-09T15:40:27,525 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:40:27,525 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 45ba5a0d25f5f40ef5c8117cd0fa3177 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:27,539 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,541 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/.tmp/testFamily/f4d47e94a6c2442ab0f4258c6dfe2682 is 61, key is row-0/testFamily:testQualifier/1731166766864/Put/seqid=0 2024-11-09T15:40:27,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741864_1040 (size=5049) 2024-11-09T15:40:27,739 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-09T15:40:27,739 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1325): Online Regions={45ba5a0d25f5f40ef5c8117cd0fa3177=testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177., ab830ca42595810345ca62bc8a3c2219=hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., 987459392592f1ceaee30b4a5bd3e7f0=hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., bd1cf2ea3c38eda30f839dfd5f4eb192=testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.} 2024-11-09T15:40:27,739 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,939 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 45ba5a0d25f5f40ef5c8117cd0fa3177, 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:27,947 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/.tmp/testFamily/f4d47e94a6c2442ab0f4258c6dfe2682 2024-11-09T15:40:27,956 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/.tmp/testFamily/f4d47e94a6c2442ab0f4258c6dfe2682 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily/f4d47e94a6c2442ab0f4258c6dfe2682 2024-11-09T15:40:27,963 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily/f4d47e94a6c2442ab0f4258c6dfe2682, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:27,965 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45ba5a0d25f5f40ef5c8117cd0fa3177 in 439ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:27,969 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextExceptionAsync/45ba5a0d25f5f40ef5c8117cd0fa3177/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:27,970 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:40:27,970 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45ba5a0d25f5f40ef5c8117cd0fa3177: Waiting for close lock at 1731166827525Running coprocessor pre-close hooks at 1731166827525Disabling compacts and flushes for region at 1731166827525Disabling writes for close at 1731166827525Obtaining lock to block concurrent updates at 1731166827525Preparing flush snapshotting stores in 45ba5a0d25f5f40ef5c8117cd0fa3177 at 1731166827525Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166827525Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. at 1731166827526 (+1 ms)Flushing 45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily: creating writer at 1731166827526Flushing 45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily: appending metadata at 1731166827541 (+15 ms)Flushing 45ba5a0d25f5f40ef5c8117cd0fa3177/testFamily: closing flushed file at 1731166827541Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b7bfceb: reopening flushed file at 1731166827954 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45ba5a0d25f5f40ef5c8117cd0fa3177 in 439ms, sequenceid=8, compaction requested=false at 1731166827965 (+11 ms)Writing region close event to WAL at 1731166827965Running coprocessor post-close hooks at 1731166827970 (+5 ms)Closed at 1731166827970 2024-11-09T15:40:27,970 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731166763761.45ba5a0d25f5f40ef5c8117cd0fa3177. 2024-11-09T15:40:27,971 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab830ca42595810345ca62bc8a3c2219, disabling compactions & flushes 2024-11-09T15:40:27,971 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:40:27,971 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:40:27,971 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. after waiting 0 ms 2024-11-09T15:40:27,971 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:40:27,971 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ab830ca42595810345ca62bc8a3c2219 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:27,987 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/.tmp/testFamily/3ba761cb4de3453da47b693c0cd8890e is 61, key is row-0/testFamily:testQualifier/1731166767999/Put/seqid=0 2024-11-09T15:40:27,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741865_1041 (size=5049) 2024-11-09T15:40:28,140 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:28,340 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 987459392592f1ceaee30b4a5bd3e7f0, ab830ca42595810345ca62bc8a3c2219, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:28,393 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/.tmp/testFamily/3ba761cb4de3453da47b693c0cd8890e 2024-11-09T15:40:28,403 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/.tmp/testFamily/3ba761cb4de3453da47b693c0cd8890e as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/testFamily/3ba761cb4de3453da47b693c0cd8890e 2024-11-09T15:40:28,411 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/testFamily/3ba761cb4de3453da47b693c0cd8890e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:28,412 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab830ca42595810345ca62bc8a3c2219 in 441ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:28,417 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnOpenScanner/ab830ca42595810345ca62bc8a3c2219/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:28,418 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab830ca42595810345ca62bc8a3c2219: Waiting for close lock at 1731166827971Running coprocessor pre-close hooks at 1731166827971Disabling compacts and flushes for region at 1731166827971Disabling writes for close at 1731166827971Obtaining lock to block concurrent updates at 1731166827971Preparing flush snapshotting stores in ab830ca42595810345ca62bc8a3c2219 at 1731166827971Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166827971Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. at 1731166827972 (+1 ms)Flushing ab830ca42595810345ca62bc8a3c2219/testFamily: creating writer at 1731166827972Flushing ab830ca42595810345ca62bc8a3c2219/testFamily: appending metadata at 1731166827987 (+15 ms)Flushing ab830ca42595810345ca62bc8a3c2219/testFamily: closing flushed file at 1731166827987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54acc71d: reopening flushed file at 1731166828402 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab830ca42595810345ca62bc8a3c2219 in 441ms, sequenceid=8, compaction requested=false at 1731166828412 (+10 ms)Writing region close event to WAL at 1731166828413 (+1 ms)Running coprocessor post-close hooks at 1731166828418 (+5 ms)Closed at 1731166828418 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731166766969.ab830ca42595810345ca62bc8a3c2219. 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 987459392592f1ceaee30b4a5bd3e7f0, disabling compactions & flushes 2024-11-09T15:40:28,418 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. after waiting 0 ms 2024-11-09T15:40:28,418 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:28,418 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 987459392592f1ceaee30b4a5bd3e7f0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:28,435 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/.tmp/testFamily/5f94250a82864b48b55f51cd62296008 is 61, key is row-0/testFamily:testQualifier/1731166812608/Put/seqid=0 2024-11-09T15:40:28,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741866_1042 (size=5049) 2024-11-09T15:40:28,540 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 987459392592f1ceaee30b4a5bd3e7f0, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:28,741 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-09T15:40:28,741 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1325): Online Regions={987459392592f1ceaee30b4a5bd3e7f0=hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., bd1cf2ea3c38eda30f839dfd5f4eb192=testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192.} 2024-11-09T15:40:28,741 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on 987459392592f1ceaee30b4a5bd3e7f0, bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:28,841 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/.tmp/testFamily/5f94250a82864b48b55f51cd62296008 2024-11-09T15:40:28,851 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/.tmp/testFamily/5f94250a82864b48b55f51cd62296008 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/testFamily/5f94250a82864b48b55f51cd62296008 2024-11-09T15:40:28,858 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/testFamily/5f94250a82864b48b55f51cd62296008, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:28,859 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 987459392592f1ceaee30b4a5bd3e7f0 in 441ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:28,864 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/hbase/testMetaScanTimeoutOnNext/987459392592f1ceaee30b4a5bd3e7f0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:28,864 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 987459392592f1ceaee30b4a5bd3e7f0: Waiting for close lock at 1731166828418Running coprocessor pre-close hooks at 1731166828418Disabling compacts and flushes for region at 1731166828418Disabling writes for close at 1731166828418Obtaining lock to block concurrent updates at 1731166828418Preparing flush snapshotting stores in 987459392592f1ceaee30b4a5bd3e7f0 at 1731166828418Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166828419 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. at 1731166828419Flushing 987459392592f1ceaee30b4a5bd3e7f0/testFamily: creating writer at 1731166828419Flushing 987459392592f1ceaee30b4a5bd3e7f0/testFamily: appending metadata at 1731166828435 (+16 ms)Flushing 987459392592f1ceaee30b4a5bd3e7f0/testFamily: closing flushed file at 1731166828435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4548e06a: reopening flushed file at 1731166828849 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 987459392592f1ceaee30b4a5bd3e7f0 in 441ms, sequenceid=8, compaction requested=false at 1731166828859 (+10 ms)Writing region close event to WAL at 1731166828860 (+1 ms)Running coprocessor post-close hooks at 1731166828864 (+4 ms)Closed at 1731166828864 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731166809560.987459392592f1ceaee30b4a5bd3e7f0. 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd1cf2ea3c38eda30f839dfd5f4eb192, disabling compactions & flushes 2024-11-09T15:40:28,865 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. after waiting 0 ms 2024-11-09T15:40:28,865 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:40:28,865 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bd1cf2ea3c38eda30f839dfd5f4eb192 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T15:40:28,881 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/.tmp/testFamily/a28618628a2e4ab2b5c08e91e3e88135 is 61, key is row-0/testFamily:testQualifier/1731166784178/Put/seqid=0 2024-11-09T15:40:28,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741867_1043 (size=5049) 2024-11-09T15:40:28,941 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:29,141 DEBUG [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1351): Waiting on bd1cf2ea3c38eda30f839dfd5f4eb192 2024-11-09T15:40:29,287 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/.tmp/testFamily/a28618628a2e4ab2b5c08e91e3e88135 2024-11-09T15:40:29,295 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/.tmp/testFamily/a28618628a2e4ab2b5c08e91e3e88135 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily/a28618628a2e4ab2b5c08e91e3e88135 2024-11-09T15:40:29,302 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily/a28618628a2e4ab2b5c08e91e3e88135, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T15:40:29,303 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd1cf2ea3c38eda30f839dfd5f4eb192 in 438ms, sequenceid=8, compaction requested=false 2024-11-09T15:40:29,308 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/data/default/testRetryOutOfOrderScannerNextException/bd1cf2ea3c38eda30f839dfd5f4eb192/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T15:40:29,309 INFO [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:40:29,309 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd1cf2ea3c38eda30f839dfd5f4eb192: Waiting for close lock at 1731166828865Running coprocessor pre-close hooks at 1731166828865Disabling compacts and flushes for region at 1731166828865Disabling writes for close at 1731166828865Obtaining lock to block concurrent updates at 1731166828865Preparing flush snapshotting stores in bd1cf2ea3c38eda30f839dfd5f4eb192 at 1731166828865Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731166828865Flushing stores of testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. at 1731166828866 (+1 ms)Flushing bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily: creating writer at 1731166828866Flushing bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily: appending metadata at 1731166828881 (+15 ms)Flushing bd1cf2ea3c38eda30f839dfd5f4eb192/testFamily: closing flushed file at 1731166828881Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4800c9c: reopening flushed file at 1731166829294 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd1cf2ea3c38eda30f839dfd5f4eb192 in 438ms, sequenceid=8, compaction requested=false at 1731166829303 (+9 ms)Writing region close event to WAL at 1731166829304 (+1 ms)Running coprocessor post-close hooks at 1731166829309 (+5 ms)Closed at 1731166829309 2024-11-09T15:40:29,309 DEBUG [RS_CLOSE_REGION-regionserver/da6b20b51b98:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731166783150.bd1cf2ea3c38eda30f839dfd5f4eb192. 2024-11-09T15:40:29,342 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(976): stopping server da6b20b51b98,45207,1731166730521; all regions closed. 2024-11-09T15:40:29,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741834_1010 (size=19479) 2024-11-09T15:40:29,352 DEBUG [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/oldWALs 2024-11-09T15:40:29,353 INFO [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da6b20b51b98%2C45207%2C1731166730521.meta:.meta(num 1731166763296) 2024-11-09T15:40:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741833_1009 (size=20031) 2024-11-09T15:40:29,357 DEBUG [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/oldWALs 2024-11-09T15:40:29,358 INFO [RS:0;da6b20b51b98:45207 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL da6b20b51b98%2C45207%2C1731166730521:(num 1731166762987) 2024-11-09T15:40:29,358 DEBUG [RS:0;da6b20b51b98:45207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T15:40:29,358 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T15:40:29,358 INFO [RS:0;da6b20b51b98:45207 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T15:40:29,358 INFO [RS:0;da6b20b51b98:45207 {}] hbase.ChoreService(370): Chore service for: regionserver/da6b20b51b98:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T15:40:29,358 INFO [RS:0;da6b20b51b98:45207 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T15:40:29,358 INFO [regionserver/da6b20b51b98:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T15:40:29,359 INFO [RS:0;da6b20b51b98:45207 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45207 2024-11-09T15:40:29,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da6b20b51b98,45207,1731166730521 2024-11-09T15:40:29,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T15:40:29,363 INFO [RS:0;da6b20b51b98:45207 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T15:40:29,365 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da6b20b51b98,45207,1731166730521] 2024-11-09T15:40:29,366 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da6b20b51b98,45207,1731166730521 already deleted, retry=false 2024-11-09T15:40:29,367 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da6b20b51b98,45207,1731166730521 expired; onlineServers=0 2024-11-09T15:40:29,367 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da6b20b51b98,34975,1731166729839' ***** 2024-11-09T15:40:29,367 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T15:40:29,367 INFO [M:0;da6b20b51b98:34975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T15:40:29,367 INFO [M:0;da6b20b51b98:34975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T15:40:29,367 DEBUG [M:0;da6b20b51b98:34975 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T15:40:29,367 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T15:40:29,367 DEBUG [M:0;da6b20b51b98:34975 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T15:40:29,367 DEBUG [master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.large.0-1731166732761 {}] cleaner.HFileCleaner(306): Exit Thread[master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.large.0-1731166732761,5,FailOnTimeoutGroup] 2024-11-09T15:40:29,367 DEBUG [master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.small.0-1731166732763 {}] cleaner.HFileCleaner(306): Exit Thread[master/da6b20b51b98:0:becomeActiveMaster-HFileCleaner.small.0-1731166732763,5,FailOnTimeoutGroup] 2024-11-09T15:40:29,367 INFO [M:0;da6b20b51b98:34975 {}] hbase.ChoreService(370): Chore service for: master/da6b20b51b98:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T15:40:29,368 INFO [M:0;da6b20b51b98:34975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T15:40:29,368 DEBUG [M:0;da6b20b51b98:34975 {}] master.HMaster(1795): Stopping service threads 2024-11-09T15:40:29,368 INFO [M:0;da6b20b51b98:34975 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T15:40:29,368 INFO [M:0;da6b20b51b98:34975 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T15:40:29,368 INFO [M:0;da6b20b51b98:34975 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T15:40:29,368 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T15:40:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T15:40:29,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T15:40:29,369 DEBUG [M:0;da6b20b51b98:34975 {}] zookeeper.ZKUtil(347): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T15:40:29,369 WARN [M:0;da6b20b51b98:34975 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T15:40:29,370 INFO [M:0;da6b20b51b98:34975 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/.lastflushedseqids 2024-11-09T15:40:29,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741868_1044 (size=712) 2024-11-09T15:40:29,392 INFO [regionserver/da6b20b51b98:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T15:40:29,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T15:40:29,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45207-0x1018daf68880001, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T15:40:29,466 INFO [RS:0;da6b20b51b98:45207 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T15:40:29,466 INFO [RS:0;da6b20b51b98:45207 {}] regionserver.HRegionServer(1031): Exiting; stopping=da6b20b51b98,45207,1731166730521; zookeeper connection closed. 2024-11-09T15:40:29,467 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@fcbc0dd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@fcbc0dd 2024-11-09T15:40:29,467 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T15:40:29,782 INFO [M:0;da6b20b51b98:34975 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T15:40:29,782 INFO [M:0;da6b20b51b98:34975 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T15:40:29,782 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T15:40:29,782 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:40:29,782 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:40:29,783 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T15:40:29,783 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:40:29,783 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.16 KB heapSize=191.18 KB 2024-11-09T15:40:29,801 DEBUG [M:0;da6b20b51b98:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d078152a37f84aab995a2b0cdb8a60e4 is 82, key is hbase:meta,,1/info:regioninfo/1731166763382/Put/seqid=0 2024-11-09T15:40:29,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741869_1045 (size=5672) 2024-11-09T15:40:30,207 INFO [M:0;da6b20b51b98:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d078152a37f84aab995a2b0cdb8a60e4 2024-11-09T15:40:30,233 DEBUG [M:0;da6b20b51b98:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8c2a15ef8e34b638fb5348952f64a0a is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731166765038/Put/seqid=0 2024-11-09T15:40:30,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741870_1046 (size=14076) 2024-11-09T15:40:30,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-09T15:40:30,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-09T15:40:30,640 INFO [M:0;da6b20b51b98:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.60 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8c2a15ef8e34b638fb5348952f64a0a 2024-11-09T15:40:30,647 INFO [M:0;da6b20b51b98:34975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f8c2a15ef8e34b638fb5348952f64a0a 2024-11-09T15:40:30,665 DEBUG [M:0;da6b20b51b98:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4271ba0d81f34c20b87ad8fa4a70ef59 is 69, key is da6b20b51b98,45207,1731166730521/rs:state/1731166732828/Put/seqid=0 2024-11-09T15:40:30,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741871_1047 (size=5156) 2024-11-09T15:40:30,783 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T15:40:31,071 INFO [M:0;da6b20b51b98:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4271ba0d81f34c20b87ad8fa4a70ef59 2024-11-09T15:40:31,078 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d078152a37f84aab995a2b0cdb8a60e4 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d078152a37f84aab995a2b0cdb8a60e4 2024-11-09T15:40:31,085 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d078152a37f84aab995a2b0cdb8a60e4, entries=8, sequenceid=319, filesize=5.5 K 2024-11-09T15:40:31,086 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8c2a15ef8e34b638fb5348952f64a0a as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f8c2a15ef8e34b638fb5348952f64a0a 2024-11-09T15:40:31,093 INFO [M:0;da6b20b51b98:34975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f8c2a15ef8e34b638fb5348952f64a0a 2024-11-09T15:40:31,093 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f8c2a15ef8e34b638fb5348952f64a0a, entries=33, sequenceid=319, filesize=13.7 K 2024-11-09T15:40:31,094 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4271ba0d81f34c20b87ad8fa4a70ef59 as hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4271ba0d81f34c20b87ad8fa4a70ef59 2024-11-09T15:40:31,100 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34413/user/jenkins/test-data/3852fcef-f537-b0df-1f14-b9d9100edd3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4271ba0d81f34c20b87ad8fa4a70ef59, entries=1, sequenceid=319, filesize=5.0 K 2024-11-09T15:40:31,101 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.88 KB/195464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=319, compaction requested=false 2024-11-09T15:40:31,103 INFO [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T15:40:31,103 DEBUG [M:0;da6b20b51b98:34975 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731166829782Disabling compacts and flushes for region at 1731166829782Disabling writes for close at 1731166829783 (+1 ms)Obtaining lock to block concurrent updates at 1731166829783Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731166829783Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167075, getHeapSize=195704, getOffHeapSize=0, getCellsCount=366 at 1731166829783Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731166829784 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731166829784Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731166829801 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731166829801Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731166830215 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731166830233 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731166830233Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731166830648 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731166830664 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731166830664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@81bbd06: reopening flushed file at 1731166831077 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690f93a9: reopening flushed file at 1731166831085 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49aca76d: reopening flushed file at 1731166831093 (+8 ms)Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.88 KB/195464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=319, compaction requested=false at 1731166831101 (+8 ms)Writing region close event to WAL at 1731166831103 (+2 ms)Closed at 1731166831103 2024-11-09T15:40:31,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741830_1006 (size=188417) 2024-11-09T15:40:31,107 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T15:40:31,107 INFO [M:0;da6b20b51b98:34975 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T15:40:31,108 INFO [M:0;da6b20b51b98:34975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34975 2024-11-09T15:40:31,108 INFO [M:0;da6b20b51b98:34975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T15:40:31,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T15:40:31,210 INFO [M:0;da6b20b51b98:34975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T15:40:31,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x1018daf68880000, quorum=127.0.0.1:53003, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T15:40:31,215 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T15:40:31,218 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T15:40:31,218 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T15:40:31,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T15:40:31,218 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.log.dir/,STOPPED} 2024-11-09T15:40:31,221 WARN [BP-702925388-172.17.0.2-1731166727042 heartbeating to localhost/127.0.0.1:34413 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T15:40:31,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T15:40:31,221 WARN [BP-702925388-172.17.0.2-1731166727042 heartbeating to localhost/127.0.0.1:34413 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-702925388-172.17.0.2-1731166727042 (Datanode Uuid c9adfbac-fbae-4e2c-8e76-c79a7f953271) service to localhost/127.0.0.1:34413 2024-11-09T15:40:31,221 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T15:40:31,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/data/data1/current/BP-702925388-172.17.0.2-1731166727042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T15:40:31,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/cluster_159f23d8-ca40-be0b-3fa6-00705e368db2/data/data2/current/BP-702925388-172.17.0.2-1731166727042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T15:40:31,223 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T15:40:31,232 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T15:40:31,233 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T15:40:31,233 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T15:40:31,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T15:40:31,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c14b7bd5-dab4-50cc-e472-173352d0c22c/hadoop.log.dir/,STOPPED} 2024-11-09T15:40:31,242 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T15:40:31,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down