2024-11-08 13:49:34,936 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-08 13:49:34,955 main DEBUG Took 0.017058 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 13:49:34,955 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 13:49:34,955 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 13:49:34,956 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 13:49:34,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,963 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 13:49:34,976 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,979 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,980 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,981 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,982 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,983 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,984 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,985 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,986 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,986 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,987 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:49:34,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,988 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 13:49:34,989 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:49:34,991 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 13:49:34,992 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 13:49:34,993 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 13:49:34,994 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 13:49:34,994 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 13:49:35,002 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 13:49:35,005 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 13:49:35,007 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 13:49:35,007 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 13:49:35,008 main DEBUG createAppenders(={Console}) 2024-11-08 13:49:35,009 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-08 13:49:35,009 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-08 13:49:35,009 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-08 13:49:35,010 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 13:49:35,010 main DEBUG OutputStream closed 2024-11-08 13:49:35,010 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 13:49:35,011 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 13:49:35,011 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-08 13:49:35,083 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 13:49:35,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 13:49:35,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 13:49:35,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 13:49:35,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 13:49:35,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 13:49:35,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 13:49:35,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 13:49:35,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 13:49:35,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 13:49:35,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 13:49:35,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 13:49:35,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 13:49:35,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 13:49:35,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 13:49:35,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 13:49:35,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 13:49:35,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 13:49:35,095 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 13:49:35,096 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-08 13:49:35,096 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 13:49:35,097 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-08T13:49:35,109 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-08 13:49:35,112 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 13:49:35,112 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T13:49:35,337 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188 2024-11-08T13:49:35,355 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T13:49:35,370 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5, deleteOnExit=true 2024-11-08T13:49:35,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T13:49:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/test.cache.data in system properties and HBase conf 2024-11-08T13:49:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T13:49:35,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.log.dir in system properties and HBase conf 2024-11-08T13:49:35,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T13:49:35,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T13:49:35,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T13:49:35,470 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T13:49:35,567 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:49:35,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T13:49:35,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T13:49:35,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T13:49:35,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T13:49:35,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T13:49:35,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T13:49:35,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T13:49:35,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T13:49:35,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T13:49:35,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/nfs.dump.dir in system properties and HBase conf 2024-11-08T13:49:35,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/java.io.tmpdir in system properties and HBase conf 2024-11-08T13:49:35,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T13:49:35,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T13:49:35,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T13:49:36,333 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T13:49:36,401 INFO [Time-limited test {}] log.Log(170): Logging initialized @2106ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T13:49:36,467 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:49:36,529 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:49:36,550 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:49:36,551 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:49:36,552 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T13:49:36,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:49:36,566 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:49:36,567 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:49:36,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/java.io.tmpdir/jetty-localhost-36155-hadoop-hdfs-3_4_1-tests_jar-_-any-11251432591293256428/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T13:49:36,752 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:36155} 2024-11-08T13:49:36,752 INFO [Time-limited test {}] server.Server(415): Started @2458ms 2024-11-08T13:49:37,069 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:49:37,077 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:49:37,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:49:37,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:49:37,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T13:49:37,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72127eee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:49:37,080 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47169f40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:49:37,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@341cfc82{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/java.io.tmpdir/jetty-localhost-34959-hadoop-hdfs-3_4_1-tests_jar-_-any-4602436603254822550/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:49:37,181 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7061d16d{HTTP/1.1, (http/1.1)}{localhost:34959} 2024-11-08T13:49:37,181 INFO [Time-limited test {}] server.Server(415): Started @2887ms 2024-11-08T13:49:37,231 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T13:49:37,673 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/data/data2/current/BP-593724743-172.17.0.2-1731073776138/current, will proceed with Du for space computation calculation, 2024-11-08T13:49:37,673 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/data/data1/current/BP-593724743-172.17.0.2-1731073776138/current, will proceed with Du for space computation calculation, 2024-11-08T13:49:37,712 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T13:49:37,771 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8dd759451082064 with lease ID 0x859eea60bbeece3f: Processing first storage report for DS-080afe3e-d602-4fba-a664-88f8ab4add74 from datanode DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138) 2024-11-08T13:49:37,773 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd759451082064 with lease ID 0x859eea60bbeece3f: from storage DS-080afe3e-d602-4fba-a664-88f8ab4add74 node DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T13:49:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8dd759451082064 with lease ID 0x859eea60bbeece3f: Processing first storage report for DS-1335aa46-62a4-47f5-aaca-0c9d04e5826c from datanode DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138) 2024-11-08T13:49:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd759451082064 with lease ID 0x859eea60bbeece3f: from storage DS-1335aa46-62a4-47f5-aaca-0c9d04e5826c node DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T13:49:37,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188 2024-11-08T13:49:37,900 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/zookeeper_0, clientPort=50109, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T13:49:37,911 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50109 2024-11-08T13:49:37,925 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:37,929 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:38,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741825_1001 (size=7) 2024-11-08T13:49:38,560 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 with version=8 2024-11-08T13:49:38,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/hbase-staging 2024-11-08T13:49:38,642 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T13:49:38,839 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6ccfea20c990:0 server-side Connection retries=9 2024-11-08T13:49:38,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:38,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:38,853 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:49:38,853 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:38,853 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:49:38,977 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T13:49:39,027 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T13:49:39,035 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T13:49:39,038 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:49:39,059 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 48727 (auto-detected) 2024-11-08T13:49:39,060 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T13:49:39,077 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43961 2024-11-08T13:49:39,097 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43961 connecting to ZooKeeper ensemble=127.0.0.1:50109 2024-11-08T13:49:39,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:439610x0, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:49:39,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43961-0x1001f98fd470000 connected 2024-11-08T13:49:39,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:39,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:39,158 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:49:39,162 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535, hbase.cluster.distributed=false 2024-11-08T13:49:39,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:49:39,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43961 2024-11-08T13:49:39,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43961 2024-11-08T13:49:39,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43961 2024-11-08T13:49:39,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43961 2024-11-08T13:49:39,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43961 2024-11-08T13:49:39,302 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=9 2024-11-08T13:49:39,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:39,304 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:39,304 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:49:39,304 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:49:39,304 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:49:39,307 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:49:39,310 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:49:39,310 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35805 2024-11-08T13:49:39,313 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35805 connecting to ZooKeeper ensemble=127.0.0.1:50109 2024-11-08T13:49:39,314 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:39,317 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:39,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358050x0, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:49:39,324 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:358050x0, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:49:39,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35805-0x1001f98fd470001 connected 2024-11-08T13:49:39,327 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:49:39,335 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:49:39,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:49:39,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:49:39,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35805 2024-11-08T13:49:39,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35805 2024-11-08T13:49:39,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35805 2024-11-08T13:49:39,348 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35805 2024-11-08T13:49:39,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35805 2024-11-08T13:49:39,368 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6ccfea20c990:43961 2024-11-08T13:49:39,369 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6ccfea20c990,43961,1731073778688 2024-11-08T13:49:39,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:49:39,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:49:39,377 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6ccfea20c990,43961,1731073778688 2024-11-08T13:49:39,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:39,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:49:39,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:39,398 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T13:49:39,400 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6ccfea20c990,43961,1731073778688 from backup master directory 2024-11-08T13:49:39,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6ccfea20c990,43961,1731073778688 2024-11-08T13:49:39,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:49:39,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:49:39,403 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:49:39,403 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6ccfea20c990,43961,1731073778688 2024-11-08T13:49:39,405 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T13:49:39,406 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T13:49:39,459 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/hbase.id] with ID: 60f124fa-d8bc-42fa-9728-849fd72cf439 2024-11-08T13:49:39,459 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/.tmp/hbase.id 2024-11-08T13:49:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741826_1002 (size=42) 2024-11-08T13:49:39,871 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/.tmp/hbase.id]:[hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/hbase.id] 2024-11-08T13:49:39,919 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:49:39,925 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T13:49:39,944 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-08T13:49:39,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:39,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:39,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741827_1003 (size=196) 2024-11-08T13:49:40,388 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:49:40,390 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T13:49:40,402 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:49:40,406 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:49:40,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741828_1004 (size=1189) 2024-11-08T13:49:40,861 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store 2024-11-08T13:49:40,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741829_1005 (size=34) 2024-11-08T13:49:41,288 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T13:49:41,294 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:49:41,295 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:49:41,296 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:49:41,296 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:49:41,298 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:49:41,298 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:49:41,298 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:49:41,299 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731073781295Disabling compacts and flushes for region at 1731073781295Disabling writes for close at 1731073781298 (+3 ms)Writing region close event to WAL at 1731073781298Closed at 1731073781298 2024-11-08T13:49:41,301 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/.initializing 2024-11-08T13:49:41,302 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/WALs/6ccfea20c990,43961,1731073778688 2024-11-08T13:49:41,309 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:49:41,323 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C43961%2C1731073778688, suffix=, logDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/WALs/6ccfea20c990,43961,1731073778688, archiveDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/oldWALs, maxLogs=10 2024-11-08T13:49:41,344 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/WALs/6ccfea20c990,43961,1731073778688/6ccfea20c990%2C43961%2C1731073778688.1731073781327, exclude list is [], retry=0 2024-11-08T13:49:41,361 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42753,DS-080afe3e-d602-4fba-a664-88f8ab4add74,DISK] 2024-11-08T13:49:41,363 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T13:49:41,394 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/WALs/6ccfea20c990,43961,1731073778688/6ccfea20c990%2C43961%2C1731073778688.1731073781327 2024-11-08T13:49:41,395 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-08T13:49:41,395 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:49:41,396 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:49:41,398 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,399 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T13:49:41,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:41,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:49:41,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T13:49:41,466 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:41,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:49:41,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T13:49:41,471 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:41,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:49:41,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T13:49:41,476 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:41,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:49:41,478 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,481 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,482 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,488 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,488 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,492 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:49:41,496 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:49:41,500 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:49:41,502 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64333400, jitterRate=-0.04135763645172119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:49:41,509 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731073781410Initializing all the Stores at 1731073781412 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073781413 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073781413Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073781414 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073781414Cleaning up temporary data from old regions at 1731073781488 (+74 ms)Region opened successfully at 1731073781509 (+21 ms) 2024-11-08T13:49:41,510 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T13:49:41,541 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1865b37f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:49:41,569 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T13:49:41,580 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T13:49:41,580 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T13:49:41,583 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:49:41,584 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:49:41,588 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T13:49:41,589 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T13:49:41,612 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T13:49:41,621 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T13:49:41,622 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T13:49:41,625 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T13:49:41,626 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T13:49:41,628 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T13:49:41,630 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T13:49:41,633 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T13:49:41,634 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T13:49:41,636 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T13:49:41,637 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T13:49:41,654 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T13:49:41,656 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T13:49:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:49:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:49:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,662 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6ccfea20c990,43961,1731073778688, sessionid=0x1001f98fd470000, setting cluster-up flag (Was=false) 2024-11-08T13:49:41,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,676 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T13:49:41,678 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,43961,1731073778688 2024-11-08T13:49:41,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:49:41,687 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T13:49:41,688 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,43961,1731073778688 2024-11-08T13:49:41,694 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T13:49:41,755 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(746): ClusterId : 60f124fa-d8bc-42fa-9728-849fd72cf439 2024-11-08T13:49:41,759 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:49:41,763 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:49:41,764 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:49:41,763 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:49:41,766 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:49:41,767 DEBUG [RS:0;6ccfea20c990:35805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4554fe5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:49:41,775 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T13:49:41,783 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T13:49:41,785 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6ccfea20c990:35805 2024-11-08T13:49:41,788 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:49:41,788 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:49:41,788 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:49:41,791 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,43961,1731073778688 with port=35805, startcode=1731073779264 2024-11-08T13:49:41,788 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6ccfea20c990,43961,1731073778688 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T13:49:41,795 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:49:41,795 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:49:41,795 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:49:41,796 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:49:41,796 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6ccfea20c990:0, corePoolSize=10, maxPoolSize=10 2024-11-08T13:49:41,796 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,796 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:49:41,796 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,800 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731073811800 2024-11-08T13:49:41,801 DEBUG [RS:0;6ccfea20c990:35805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:49:41,801 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T13:49:41,802 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:49:41,803 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T13:49:41,803 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T13:49:41,806 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T13:49:41,807 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T13:49:41,807 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T13:49:41,807 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T13:49:41,808 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,810 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:41,811 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T13:49:41,810 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T13:49:41,812 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T13:49:41,812 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T13:49:41,814 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T13:49:41,815 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T13:49:41,817 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731073781816,5,FailOnTimeoutGroup] 2024-11-08T13:49:41,820 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731073781817,5,FailOnTimeoutGroup] 2024-11-08T13:49:41,821 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,821 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T13:49:41,822 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,823 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741831_1007 (size=1321) 2024-11-08T13:49:41,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54735, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:49:41,844 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43961 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43961 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,859 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:49:41,859 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44643 2024-11-08T13:49:41,860 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:49:41,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:49:41,864 DEBUG [RS:0;6ccfea20c990:35805 {}] zookeeper.ZKUtil(111): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,865 WARN [RS:0;6ccfea20c990:35805 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:49:41,865 INFO [RS:0;6ccfea20c990:35805 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:49:41,865 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,867 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,35805,1731073779264] 2024-11-08T13:49:41,890 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:49:41,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:49:41,913 INFO [RS:0;6ccfea20c990:35805 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:49:41,913 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,914 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:49:41,920 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:49:41,921 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,921 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:49:41,922 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:49:41,923 DEBUG [RS:0;6ccfea20c990:35805 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,925 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35805,1731073779264-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:49:41,944 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:49:41,946 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35805,1731073779264-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,947 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,947 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.Replication(171): 6ccfea20c990,35805,1731073779264 started 2024-11-08T13:49:41,966 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:49:41,966 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,35805,1731073779264, RpcServer on 6ccfea20c990/172.17.0.2:35805, sessionid=0x1001f98fd470001 2024-11-08T13:49:41,967 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:49:41,967 DEBUG [RS:0;6ccfea20c990:35805 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,968 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35805,1731073779264' 2024-11-08T13:49:41,968 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:49:41,969 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:49:41,970 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:49:41,970 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:49:41,970 DEBUG [RS:0;6ccfea20c990:35805 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,35805,1731073779264 2024-11-08T13:49:41,970 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35805,1731073779264' 2024-11-08T13:49:41,970 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:49:41,971 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:49:41,972 DEBUG [RS:0;6ccfea20c990:35805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:49:41,972 INFO [RS:0;6ccfea20c990:35805 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:49:41,972 INFO [RS:0;6ccfea20c990:35805 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:49:42,230 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T13:49:42,231 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:49:42,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741832_1008 (size=32) 2024-11-08T13:49:42,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:49:42,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T13:49:42,658 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:49:42,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:42,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:49:42,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T13:49:42,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:49:42,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:42,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:49:42,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T13:49:42,665 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:49:42,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:42,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:49:42,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T13:49:42,669 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:49:42,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:49:42,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:49:42,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:49:42,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740 2024-11-08T13:49:42,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740 2024-11-08T13:49:42,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:49:42,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:49:42,678 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:49:42,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:49:42,684 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:49:42,685 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68833784, jitterRate=0.0257033109664917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:49:42,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731073782651Initializing all the Stores at 1731073782655 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073782655Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073782655Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073782655Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073782655Cleaning up temporary data from old regions at 1731073782677 (+22 ms)Region opened successfully at 1731073782689 (+12 ms) 2024-11-08T13:49:42,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:49:42,689 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:49:42,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:49:42,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:49:42,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:49:42,691 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:49:42,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731073782689Disabling compacts and flushes for region at 1731073782689Disabling writes for close at 1731073782690 (+1 ms)Writing region close event to WAL at 1731073782691 (+1 ms)Closed at 1731073782691 2024-11-08T13:49:42,695 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:49:42,695 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T13:49:42,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:49:42,708 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:49:42,711 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T13:49:42,863 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:43,865 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:44,866 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:45,868 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:46,869 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:47,870 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:48,871 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:49,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T13:49:49,027 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T13:49:49,872 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:50,873 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:51,875 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:52,876 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:53,877 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:54,878 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:55,879 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:56,880 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:57,882 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:58,883 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:49:59,884 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:00,885 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:01,886 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:02,887 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:03,889 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:04,890 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:05,891 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:06,892 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:07,829 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:50:07,893 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:08,894 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:09,895 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:10,896 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:11,897 WARN [6ccfea20c990:43961 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:50:11,986 INFO [RS:0;6ccfea20c990:35805 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:50:11,990 INFO [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C35805%2C1731073779264, suffix=, logDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264, archiveDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/oldWALs, maxLogs=32 2024-11-08T13:50:12,005 DEBUG [RS:0;6ccfea20c990:35805 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264/6ccfea20c990%2C35805%2C1731073779264.1731073811992, exclude list is [], retry=0 2024-11-08T13:50:12,009 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42753,DS-080afe3e-d602-4fba-a664-88f8ab4add74,DISK] 2024-11-08T13:50:12,012 INFO [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264/6ccfea20c990%2C35805%2C1731073779264.1731073811992 2024-11-08T13:50:12,013 DEBUG [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-08T13:50:12,154 DEBUG [6ccfea20c990:43961 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T13:50:12,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:12,174 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35805,1731073779264, state=OPENING 2024-11-08T13:50:12,178 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T13:50:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:50:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:50:12,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:50:12,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:50:12,181 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:50:12,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:12,364 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:50:12,367 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:50:12,377 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:50:12,378 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:50:12,379 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:50:12,382 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C35805%2C1731073779264.meta, suffix=.meta, logDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264, archiveDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/oldWALs, maxLogs=32 2024-11-08T13:50:12,402 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264/6ccfea20c990%2C35805%2C1731073779264.meta.1731073812384.meta, exclude list is [], retry=0 2024-11-08T13:50:12,405 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42753,DS-080afe3e-d602-4fba-a664-88f8ab4add74,DISK] 2024-11-08T13:50:12,409 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264/6ccfea20c990%2C35805%2C1731073779264.meta.1731073812384.meta 2024-11-08T13:50:12,409 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-08T13:50:12,409 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:12,411 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:50:12,413 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:50:12,418 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:50:12,421 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:50:12,422 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:12,422 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:50:12,422 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:50:12,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T13:50:12,427 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:50:12,427 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:12,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:50:12,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T13:50:12,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:50:12,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:12,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:50:12,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T13:50:12,432 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:50:12,432 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:12,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:50:12,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T13:50:12,434 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:50:12,434 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:12,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:50:12,435 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:50:12,437 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740 2024-11-08T13:50:12,440 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740 2024-11-08T13:50:12,443 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:50:12,443 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:50:12,444 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:50:12,447 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:50:12,449 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71199367, jitterRate=0.06095324456691742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:50:12,449 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:50:12,451 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731073812423Writing region info on filesystem at 1731073812423Initializing all the Stores at 1731073812425 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073812425Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073812425Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073812425Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731073812425Cleaning up temporary data from old regions at 1731073812443 (+18 ms)Running coprocessor post-open hooks at 1731073812449 (+6 ms)Region opened successfully at 1731073812450 (+1 ms) 2024-11-08T13:50:12,457 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731073812357 2024-11-08T13:50:12,470 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:50:12,470 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:50:12,473 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:12,476 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35805,1731073779264, state=OPEN 2024-11-08T13:50:12,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:50:12,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:50:12,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:50:12,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:50:12,480 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:12,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T13:50:12,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35805,1731073779264 in 297 msec 2024-11-08T13:50:12,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T13:50:12,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.7880 sec 2024-11-08T13:50:12,496 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:50:12,496 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T13:50:12,512 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:50:12,514 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35805,1731073779264, seqNum=-1] 2024-11-08T13:50:12,534 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:50:12,537 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39531, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:50:12,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8350 sec 2024-11-08T13:50:12,560 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731073812560, completionTime=-1 2024-11-08T13:50:12,563 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T13:50:12,563 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:50:12,591 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-08T13:50:12,591 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731073872591 2024-11-08T13:50:12,591 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731073932591 2024-11-08T13:50:12,591 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-08T13:50:12,593 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,594 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,594 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,595 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6ccfea20c990:43961, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,595 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,596 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,603 DEBUG [master/6ccfea20c990:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T13:50:12,625 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.221sec 2024-11-08T13:50:12,626 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T13:50:12,628 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T13:50:12,629 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T13:50:12,630 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T13:50:12,630 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T13:50:12,631 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:50:12,631 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T13:50:12,640 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T13:50:12,640 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T13:50:12,641 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,43961,1731073778688-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:50:12,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e426303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,709 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T13:50:12,709 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T13:50:12,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ccfea20c990,43961,-1 for getting cluster id 2024-11-08T13:50:12,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T13:50:12,724 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '60f124fa-d8bc-42fa-9728-849fd72cf439' 2024-11-08T13:50:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T13:50:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "60f124fa-d8bc-42fa-9728-849fd72cf439" 2024-11-08T13:50:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d3f08f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ccfea20c990,43961,-1] 2024-11-08T13:50:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T13:50:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:50:12,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:50:12,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b51c427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:50:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35805,1731073779264, seqNum=-1] 2024-11-08T13:50:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:50:12,746 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:50:12,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6ccfea20c990,43961,1731073778688 2024-11-08T13:50:12,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1cdb4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ccfea20c990,43961,-1 for getting cluster id 2024-11-08T13:50:12,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T13:50:12,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '60f124fa-d8bc-42fa-9728-849fd72cf439' 2024-11-08T13:50:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T13:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "60f124fa-d8bc-42fa-9728-849fd72cf439" 2024-11-08T13:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200ed736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ccfea20c990,43961,-1] 2024-11-08T13:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T13:50:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:50:12,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:50:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c140d85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:50:12,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=52, ProcessCount=11, AvailableMemoryMB=3417 2024-11-08T13:50:12,792 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T13:50:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6ccfea20c990,43961,1731073778688 2024-11-08T13:50:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53a284bf 2024-11-08T13:50:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T13:50:12,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T13:50:12,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-08T13:50:12,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:12,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-08T13:50:12,838 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:12,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:50:12,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741835_1011 (size=428) 2024-11-08T13:50:13,266 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7095e804fef2690ab6580c0941d14ab7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:13,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741836_1012 (size=79) 2024-11-08T13:50:13,679 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:13,679 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7095e804fef2690ab6580c0941d14ab7, disabling compactions & flushes 2024-11-08T13:50:13,680 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:13,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:13,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. after waiting 0 ms 2024-11-08T13:50:13,680 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:13,681 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:13,681 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7095e804fef2690ab6580c0941d14ab7: Waiting for close lock at 1731073813679Disabling compacts and flushes for region at 1731073813679Disabling writes for close at 1731073813680 (+1 ms)Writing region close event to WAL at 1731073813680Closed at 1731073813680 2024-11-08T13:50:13,684 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:13,689 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731073813684"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073813684"}]},"ts":"1731073813684"} 2024-11-08T13:50:13,694 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:13,695 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:13,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073813695"}]},"ts":"1731073813695"} 2024-11-08T13:50:13,702 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-08T13:50:13,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7095e804fef2690ab6580c0941d14ab7, ASSIGN}] 2024-11-08T13:50:13,706 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7095e804fef2690ab6580c0941d14ab7, ASSIGN 2024-11-08T13:50:13,708 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7095e804fef2690ab6580c0941d14ab7, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:13,862 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7095e804fef2690ab6580c0941d14ab7, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:50:13,870 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7095e804fef2690ab6580c0941d14ab7, ASSIGN because future has completed 2024-11-08T13:50:13,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7095e804fef2690ab6580c0941d14ab7, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:14,040 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:14,041 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7095e804fef2690ab6580c0941d14ab7, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:14,041 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,041 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:14,041 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,041 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,044 INFO [StoreOpener-7095e804fef2690ab6580c0941d14ab7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,046 INFO [StoreOpener-7095e804fef2690ab6580c0941d14ab7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7095e804fef2690ab6580c0941d14ab7 columnFamilyName testFamily 2024-11-08T13:50:14,046 DEBUG [StoreOpener-7095e804fef2690ab6580c0941d14ab7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:14,047 INFO [StoreOpener-7095e804fef2690ab6580c0941d14ab7-1 {}] regionserver.HStore(327): Store=7095e804fef2690ab6580c0941d14ab7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:14,047 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,049 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,049 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,050 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,050 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,053 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,056 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:14,057 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7095e804fef2690ab6580c0941d14ab7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72777377, jitterRate=0.08446742594242096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:14,057 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:50:14,058 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7095e804fef2690ab6580c0941d14ab7: Running coprocessor pre-open hook at 1731073814042Writing region info on filesystem at 1731073814042Initializing all the Stores at 1731073814043 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073814043Cleaning up temporary data from old regions at 1731073814050 (+7 ms)Running coprocessor post-open hooks at 1731073814057 (+7 ms)Region opened successfully at 1731073814058 (+1 ms) 2024-11-08T13:50:14,060 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7., pid=6, masterSystemTime=1731073814026 2024-11-08T13:50:14,063 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:14,063 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:50:14,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7095e804fef2690ab6580c0941d14ab7, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:14,069 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7095e804fef2690ab6580c0941d14ab7, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:14,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T13:50:14,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7095e804fef2690ab6580c0941d14ab7, server=6ccfea20c990,35805,1731073779264 in 201 msec 2024-11-08T13:50:14,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T13:50:14,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=7095e804fef2690ab6580c0941d14ab7, ASSIGN in 371 msec 2024-11-08T13:50:14,082 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:14,082 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073814082"}]},"ts":"1731073814082"} 2024-11-08T13:50:14,085 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-08T13:50:14,087 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:14,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2760 sec 2024-11-08T13:50:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:50:15,896 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-08T13:50:15,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-08T13:50:15,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:15,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-08T13:50:15,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:15,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-08T13:50:15,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:50:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35805,1731073779264, seqNum=-1] 2024-11-08T13:50:15,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:50:15,917 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:50:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:15,938 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:15,941 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-08T13:50:15,946 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T13:50:15,952 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T13:50:15,954 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T13:50:15,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=570217772525551624 2024-11-08T13:50:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073818954, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:15,982 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-08T13:50:15,983 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-08T13:50:15,985 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:15,986 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:15,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=570217772525551626 2024-11-08T13:50:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073818986, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:15,991 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:15,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=570217772525551627 2024-11-08T13:50:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073818990, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:15,995 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:15,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=570217772525551628 2024-11-08T13:50:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073818995, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:16,000 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:16,012 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1242505112_22 at /127.0.0.1:45864 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=52 (was 52), ProcessCount=11 (was 11), AvailableMemoryMB=3369 (was 3417) 2024-11-08T13:50:16,021 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=52, ProcessCount=11, AvailableMemoryMB=3368 2024-11-08T13:50:16,024 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:16,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-08T13:50:16,028 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:16,029 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:16,029 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-08T13:50:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T13:50:16,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:16,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741837_1013 (size=414) 2024-11-08T13:50:16,448 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3c9deaa9eec2c9ef9815dffeb3e01cc, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:16,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741838_1014 (size=65) 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing a3c9deaa9eec2c9ef9815dffeb3e01cc, disabling compactions & flushes 2024-11-08T13:50:16,462 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. after waiting 0 ms 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,462 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,462 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3c9deaa9eec2c9ef9815dffeb3e01cc: Waiting for close lock at 1731073816462Disabling compacts and flushes for region at 1731073816462Disabling writes for close at 1731073816462Writing region close event to WAL at 1731073816462Closed at 1731073816462 2024-11-08T13:50:16,466 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:16,466 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731073816466"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073816466"}]},"ts":"1731073816466"} 2024-11-08T13:50:16,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:16,473 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:16,473 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073816473"}]},"ts":"1731073816473"} 2024-11-08T13:50:16,476 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-08T13:50:16,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a3c9deaa9eec2c9ef9815dffeb3e01cc, ASSIGN}] 2024-11-08T13:50:16,479 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a3c9deaa9eec2c9ef9815dffeb3e01cc, ASSIGN 2024-11-08T13:50:16,480 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a3c9deaa9eec2c9ef9815dffeb3e01cc, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:16,631 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a3c9deaa9eec2c9ef9815dffeb3e01cc, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:16,635 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a3c9deaa9eec2c9ef9815dffeb3e01cc, ASSIGN because future has completed 2024-11-08T13:50:16,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3c9deaa9eec2c9ef9815dffeb3e01cc, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:16,805 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a3c9deaa9eec2c9ef9815dffeb3e01cc, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:16,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:16,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,806 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,808 INFO [StoreOpener-a3c9deaa9eec2c9ef9815dffeb3e01cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,810 INFO [StoreOpener-a3c9deaa9eec2c9ef9815dffeb3e01cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3c9deaa9eec2c9ef9815dffeb3e01cc columnFamilyName testFamily 2024-11-08T13:50:16,810 DEBUG [StoreOpener-a3c9deaa9eec2c9ef9815dffeb3e01cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:16,811 INFO [StoreOpener-a3c9deaa9eec2c9ef9815dffeb3e01cc-1 {}] regionserver.HStore(327): Store=a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:16,811 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,812 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,813 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,814 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,814 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,816 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,819 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:16,820 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened a3c9deaa9eec2c9ef9815dffeb3e01cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70879520, jitterRate=0.05618715286254883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:16,820 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:50:16,822 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a3c9deaa9eec2c9ef9815dffeb3e01cc: Running coprocessor pre-open hook at 1731073816806Writing region info on filesystem at 1731073816806Initializing all the Stores at 1731073816808 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073816808Cleaning up temporary data from old regions at 1731073816814 (+6 ms)Running coprocessor post-open hooks at 1731073816821 (+7 ms)Region opened successfully at 1731073816821 2024-11-08T13:50:16,823 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., pid=9, masterSystemTime=1731073816791 2024-11-08T13:50:16,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,827 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:50:16,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a3c9deaa9eec2c9ef9815dffeb3e01cc, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:16,831 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3c9deaa9eec2c9ef9815dffeb3e01cc, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:16,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T13:50:16,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure a3c9deaa9eec2c9ef9815dffeb3e01cc, server=6ccfea20c990,35805,1731073779264 in 197 msec 2024-11-08T13:50:16,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T13:50:16,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a3c9deaa9eec2c9ef9815dffeb3e01cc, ASSIGN in 361 msec 2024-11-08T13:50:16,842 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:16,842 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073816842"}]},"ts":"1731073816842"} 2024-11-08T13:50:16,845 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-08T13:50:16,847 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:16,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 822 msec 2024-11-08T13:50:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T13:50:17,053 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-08T13:50:17,053 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-08T13:50:17,054 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:17,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-08T13:50:17,062 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:17,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-08T13:50:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:17,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:17,079 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-08T13:50:17,083 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T13:50:17,085 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-08T13:50:17,963 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:50:17,990 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-08T13:50:17,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T13:50:17,992 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-08T13:50:19,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T13:50:19,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:19,028 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:50:19,028 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T13:50:19,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-08T13:50:19,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:19,033 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-08T13:50:19,033 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:23,101 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-08T13:50:23,102 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-11-08T13:50:23,102 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:50:23,110 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=73 (was 52) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3359 (was 3368) 2024-11-08T13:50:23,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=73, ProcessCount=11, AvailableMemoryMB=3359 2024-11-08T13:50:23,122 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:23,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-08T13:50:23,126 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:23,126 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:23,127 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-08T13:50:23,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:23,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-08T13:50:23,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741839_1015 (size=412) 2024-11-08T13:50:23,545 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7da3d795ef6a7df5fedbbfe4ac343af0, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741840_1016 (size=63) 2024-11-08T13:50:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-08T13:50:23,962 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:23,962 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 7da3d795ef6a7df5fedbbfe4ac343af0, disabling compactions & flushes 2024-11-08T13:50:23,963 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:23,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:23,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. after waiting 0 ms 2024-11-08T13:50:23,963 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:23,963 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:23,964 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7da3d795ef6a7df5fedbbfe4ac343af0: Waiting for close lock at 1731073823962Disabling compacts and flushes for region at 1731073823962Disabling writes for close at 1731073823963 (+1 ms)Writing region close event to WAL at 1731073823963Closed at 1731073823963 2024-11-08T13:50:23,969 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:23,969 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731073823969"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073823969"}]},"ts":"1731073823969"} 2024-11-08T13:50:23,972 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:23,974 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:23,974 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073823974"}]},"ts":"1731073823974"} 2024-11-08T13:50:23,976 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-08T13:50:23,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=7da3d795ef6a7df5fedbbfe4ac343af0, ASSIGN}] 2024-11-08T13:50:23,978 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=7da3d795ef6a7df5fedbbfe4ac343af0, ASSIGN 2024-11-08T13:50:23,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=7da3d795ef6a7df5fedbbfe4ac343af0, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:24,131 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=7da3d795ef6a7df5fedbbfe4ac343af0, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:24,140 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=7da3d795ef6a7df5fedbbfe4ac343af0, ASSIGN because future has completed 2024-11-08T13:50:24,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7da3d795ef6a7df5fedbbfe4ac343af0, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-08T13:50:24,305 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:24,305 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 7da3d795ef6a7df5fedbbfe4ac343af0, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:24,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:24,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,307 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,310 INFO [StoreOpener-7da3d795ef6a7df5fedbbfe4ac343af0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,313 INFO [StoreOpener-7da3d795ef6a7df5fedbbfe4ac343af0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7da3d795ef6a7df5fedbbfe4ac343af0 columnFamilyName testFamily 2024-11-08T13:50:24,313 DEBUG [StoreOpener-7da3d795ef6a7df5fedbbfe4ac343af0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:24,314 INFO [StoreOpener-7da3d795ef6a7df5fedbbfe4ac343af0-1 {}] regionserver.HStore(327): Store=7da3d795ef6a7df5fedbbfe4ac343af0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:24,314 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,315 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,316 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,319 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,322 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:24,323 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 7da3d795ef6a7df5fedbbfe4ac343af0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74460898, jitterRate=0.10955384373664856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:24,323 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:50:24,324 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 7da3d795ef6a7df5fedbbfe4ac343af0: Running coprocessor pre-open hook at 1731073824307Writing region info on filesystem at 1731073824307Initializing all the Stores at 1731073824310 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073824310Cleaning up temporary data from old regions at 1731073824316 (+6 ms)Running coprocessor post-open hooks at 1731073824323 (+7 ms)Region opened successfully at 1731073824324 (+1 ms) 2024-11-08T13:50:24,325 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0., pid=12, masterSystemTime=1731073824294 2024-11-08T13:50:24,329 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:24,329 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:50:24,330 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=7da3d795ef6a7df5fedbbfe4ac343af0, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:24,333 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7da3d795ef6a7df5fedbbfe4ac343af0, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:24,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-08T13:50:24,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 7da3d795ef6a7df5fedbbfe4ac343af0, server=6ccfea20c990,35805,1731073779264 in 195 msec 2024-11-08T13:50:24,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T13:50:24,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=7da3d795ef6a7df5fedbbfe4ac343af0, ASSIGN in 362 msec 2024-11-08T13:50:24,343 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:24,343 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073824343"}]},"ts":"1731073824343"} 2024-11-08T13:50:24,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-08T13:50:24,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:24,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2250 sec 2024-11-08T13:50:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-08T13:50:26,151 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-08T13:50:26,151 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-08T13:50:26,151 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:26,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-08T13:50:26,155 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:26,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-08T13:50:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:26,170 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:26,170 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-08T13:50:26,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T13:50:26,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T13:50:26,177 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-08T13:50:26,178 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-08T13:50:29,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-08T13:50:29,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:29,833 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:50:29,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-08T13:50:32,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551637' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T13:50:32,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:53022 deadline: 1731073838189, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551637' does not exist 2024-11-08T13:50:32,199 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 6ccfea20c990,35805,1731073779264 for closing scanner id = 570217772525551637 for 7da3d795ef6a7df5fedbbfe4ac343af0 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551637' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:50:32,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=62 (was 73), ProcessCount=11 (was 11), AvailableMemoryMB=3353 (was 3359) 2024-11-08T13:50:32,211 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=62, ProcessCount=11, AvailableMemoryMB=3352 2024-11-08T13:50:32,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-08T13:50:32,216 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:32,216 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:32,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-08T13:50:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T13:50:32,218 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:32,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741841_1017 (size=423) 2024-11-08T13:50:32,634 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ea36ea2c345bf3b636ce12c5dbf3000, NAME => 'testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:32,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741842_1018 (size=74) 2024-11-08T13:50:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-08T13:50:33,049 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:33,050 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 1ea36ea2c345bf3b636ce12c5dbf3000, disabling compactions & flushes 2024-11-08T13:50:33,050 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,050 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,050 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. after waiting 0 ms 2024-11-08T13:50:33,050 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,051 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,051 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ea36ea2c345bf3b636ce12c5dbf3000: Waiting for close lock at 1731073833050Disabling compacts and flushes for region at 1731073833050Disabling writes for close at 1731073833050Writing region close event to WAL at 1731073833051 (+1 ms)Closed at 1731073833051 2024-11-08T13:50:33,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:33,058 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731073833057"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073833057"}]},"ts":"1731073833057"} 2024-11-08T13:50:33,062 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:33,064 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:33,064 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073833064"}]},"ts":"1731073833064"} 2024-11-08T13:50:33,066 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-08T13:50:33,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1ea36ea2c345bf3b636ce12c5dbf3000, ASSIGN}] 2024-11-08T13:50:33,069 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1ea36ea2c345bf3b636ce12c5dbf3000, ASSIGN 2024-11-08T13:50:33,070 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1ea36ea2c345bf3b636ce12c5dbf3000, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:33,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1ea36ea2c345bf3b636ce12c5dbf3000, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:33,230 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1ea36ea2c345bf3b636ce12c5dbf3000, ASSIGN because future has completed 2024-11-08T13:50:33,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ea36ea2c345bf3b636ce12c5dbf3000, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T13:50:33,395 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,396 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ea36ea2c345bf3b636ce12c5dbf3000, NAME => 'testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:33,397 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,397 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:33,397 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,397 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,401 INFO [StoreOpener-1ea36ea2c345bf3b636ce12c5dbf3000-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,406 INFO [StoreOpener-1ea36ea2c345bf3b636ce12c5dbf3000-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ea36ea2c345bf3b636ce12c5dbf3000 columnFamilyName testFamily 2024-11-08T13:50:33,406 DEBUG [StoreOpener-1ea36ea2c345bf3b636ce12c5dbf3000-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:33,407 INFO [StoreOpener-1ea36ea2c345bf3b636ce12c5dbf3000-1 {}] regionserver.HStore(327): Store=1ea36ea2c345bf3b636ce12c5dbf3000/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:33,407 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,409 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,410 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,411 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,411 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,414 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,418 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:33,419 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 1ea36ea2c345bf3b636ce12c5dbf3000; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64171825, jitterRate=-0.043765291571617126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:33,419 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:50:33,420 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 1ea36ea2c345bf3b636ce12c5dbf3000: Running coprocessor pre-open hook at 1731073833397Writing region info on filesystem at 1731073833397Initializing all the Stores at 1731073833400 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073833400Cleaning up temporary data from old regions at 1731073833411 (+11 ms)Running coprocessor post-open hooks at 1731073833419 (+8 ms)Region opened successfully at 1731073833420 (+1 ms) 2024-11-08T13:50:33,421 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., pid=15, masterSystemTime=1731073833385 2024-11-08T13:50:33,424 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,424 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:50:33,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1ea36ea2c345bf3b636ce12c5dbf3000, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:33,428 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ea36ea2c345bf3b636ce12c5dbf3000, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:33,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-08T13:50:33,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ea36ea2c345bf3b636ce12c5dbf3000, server=6ccfea20c990,35805,1731073779264 in 199 msec 2024-11-08T13:50:33,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T13:50:33,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1ea36ea2c345bf3b636ce12c5dbf3000, ASSIGN in 366 msec 2024-11-08T13:50:33,438 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:33,438 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073833438"}]},"ts":"1731073833438"} 2024-11-08T13:50:33,440 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-08T13:50:33,442 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:33,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2290 sec 2024-11-08T13:50:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T13:50:35,254 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-08T13:50:35,254 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-08T13:50:35,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:35,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-08T13:50:35,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:35,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-08T13:50:35,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:35,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:35,280 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-08T13:50:35,284 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-08T13:50:35,287 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-08T13:50:35,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-08T13:50:35,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=570217772525551641 2024-11-08T13:50:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073838289, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:35,295 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-08T13:50:35,297 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-08T13:50:35,298 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:35,300 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:35,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=570217772525551643 2024-11-08T13:50:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073838299, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:35,305 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:35,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=570217772525551644 2024-11-08T13:50:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073838304, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:35,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:35,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=570217772525551645 2024-11-08T13:50:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:53022 deadline: 1731073838309, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-08T13:50:35,314 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-08T13:50:35,321 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1242505112_22 at /127.0.0.1:53424 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=62 (was 62), ProcessCount=11 (was 11), AvailableMemoryMB=3349 (was 3352) 2024-11-08T13:50:35,329 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=62, ProcessCount=11, AvailableMemoryMB=3349 2024-11-08T13:50:35,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-08T13:50:35,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:35,335 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:35,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-08T13:50:35,337 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-08T13:50:35,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741843_1019 (size=418) 2024-11-08T13:50:35,754 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b95944d9363e6561b6ea9b48f200d832, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741844_1020 (size=69) 2024-11-08T13:50:35,765 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:35,765 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing b95944d9363e6561b6ea9b48f200d832, disabling compactions & flushes 2024-11-08T13:50:35,765 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:35,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:35,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. after waiting 0 ms 2024-11-08T13:50:35,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:35,766 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:35,766 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for b95944d9363e6561b6ea9b48f200d832: Waiting for close lock at 1731073835765Disabling compacts and flushes for region at 1731073835765Disabling writes for close at 1731073835766 (+1 ms)Writing region close event to WAL at 1731073835766Closed at 1731073835766 2024-11-08T13:50:35,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:35,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731073835767"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073835767"}]},"ts":"1731073835767"} 2024-11-08T13:50:35,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:35,772 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:35,772 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073835772"}]},"ts":"1731073835772"} 2024-11-08T13:50:35,774 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-08T13:50:35,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b95944d9363e6561b6ea9b48f200d832, ASSIGN}] 2024-11-08T13:50:35,776 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b95944d9363e6561b6ea9b48f200d832, ASSIGN 2024-11-08T13:50:35,778 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b95944d9363e6561b6ea9b48f200d832, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:35,929 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b95944d9363e6561b6ea9b48f200d832, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:35,936 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b95944d9363e6561b6ea9b48f200d832, ASSIGN because future has completed 2024-11-08T13:50:35,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b95944d9363e6561b6ea9b48f200d832, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:35,996 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-08T13:50:36,100 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:36,100 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => b95944d9363e6561b6ea9b48f200d832, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:36,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:36,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,102 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,105 INFO [StoreOpener-b95944d9363e6561b6ea9b48f200d832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,107 INFO [StoreOpener-b95944d9363e6561b6ea9b48f200d832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b95944d9363e6561b6ea9b48f200d832 columnFamilyName testFamily 2024-11-08T13:50:36,107 DEBUG [StoreOpener-b95944d9363e6561b6ea9b48f200d832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:36,108 INFO [StoreOpener-b95944d9363e6561b6ea9b48f200d832-1 {}] regionserver.HStore(327): Store=b95944d9363e6561b6ea9b48f200d832/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:36,108 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,109 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,110 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,111 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,111 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,113 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,117 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:36,118 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened b95944d9363e6561b6ea9b48f200d832; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68454782, jitterRate=0.02005574107170105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:36,118 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:50:36,119 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for b95944d9363e6561b6ea9b48f200d832: Running coprocessor pre-open hook at 1731073836102Writing region info on filesystem at 1731073836102Initializing all the Stores at 1731073836104 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073836104Cleaning up temporary data from old regions at 1731073836111 (+7 ms)Running coprocessor post-open hooks at 1731073836118 (+7 ms)Region opened successfully at 1731073836119 (+1 ms) 2024-11-08T13:50:36,120 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., pid=18, masterSystemTime=1731073836091 2024-11-08T13:50:36,123 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:36,123 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:50:36,124 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b95944d9363e6561b6ea9b48f200d832, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:36,127 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b95944d9363e6561b6ea9b48f200d832, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:36,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-08T13:50:36,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure b95944d9363e6561b6ea9b48f200d832, server=6ccfea20c990,35805,1731073779264 in 191 msec 2024-11-08T13:50:36,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T13:50:36,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=b95944d9363e6561b6ea9b48f200d832, ASSIGN in 356 msec 2024-11-08T13:50:36,136 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:36,136 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073836136"}]},"ts":"1731073836136"} 2024-11-08T13:50:36,139 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-08T13:50:36,140 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:36,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 808 msec 2024-11-08T13:50:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-08T13:50:36,353 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-08T13:50:36,353 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-08T13:50:36,354 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:36,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-08T13:50:36,365 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:36,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-08T13:50:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:36,380 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:36,381 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-08T13:50:36,384 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T13:50:36,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-08T13:50:37,386 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-08T13:50:37,386 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-08T13:50:37,387 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:50:37,830 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:50:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-08T13:50:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:38,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-08T13:50:38,904 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:50:39,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-08T13:50:39,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:39,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-08T13:50:39,025 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:39,387 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=978ms 2024-11-08T13:50:39,387 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=978ms 2024-11-08T13:50:39,387 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:50:39,403 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 228), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=57 (was 62), ProcessCount=11 (was 11), AvailableMemoryMB=3344 (was 3349) 2024-11-08T13:50:39,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=57, ProcessCount=11, AvailableMemoryMB=3343 2024-11-08T13:50:39,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-08T13:50:39,419 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:39,420 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:39,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-08T13:50:39,421 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T13:50:39,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741845_1021 (size=416) 2024-11-08T13:50:39,838 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f43b241499a6c8340b04792ca985f07b, NAME => 'testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:39,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741846_1022 (size=67) 2024-11-08T13:50:39,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-08T13:50:40,253 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:40,253 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f43b241499a6c8340b04792ca985f07b, disabling compactions & flushes 2024-11-08T13:50:40,253 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. after waiting 0 ms 2024-11-08T13:50:40,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,254 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,255 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f43b241499a6c8340b04792ca985f07b: Waiting for close lock at 1731073840253Disabling compacts and flushes for region at 1731073840253Disabling writes for close at 1731073840254 (+1 ms)Writing region close event to WAL at 1731073840254Closed at 1731073840254 2024-11-08T13:50:40,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:40,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731073840261"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073840261"}]},"ts":"1731073840261"} 2024-11-08T13:50:40,265 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:40,267 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:40,267 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073840267"}]},"ts":"1731073840267"} 2024-11-08T13:50:40,270 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-08T13:50:40,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f43b241499a6c8340b04792ca985f07b, ASSIGN}] 2024-11-08T13:50:40,272 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f43b241499a6c8340b04792ca985f07b, ASSIGN 2024-11-08T13:50:40,273 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f43b241499a6c8340b04792ca985f07b, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:40,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f43b241499a6c8340b04792ca985f07b, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T13:50:40,432 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f43b241499a6c8340b04792ca985f07b, ASSIGN because future has completed 2024-11-08T13:50:40,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43b241499a6c8340b04792ca985f07b, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:40,599 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,599 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => f43b241499a6c8340b04792ca985f07b, NAME => 'testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:40,600 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,600 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:40,600 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,600 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,602 INFO [StoreOpener-f43b241499a6c8340b04792ca985f07b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,604 INFO [StoreOpener-f43b241499a6c8340b04792ca985f07b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43b241499a6c8340b04792ca985f07b columnFamilyName testFamily 2024-11-08T13:50:40,605 DEBUG [StoreOpener-f43b241499a6c8340b04792ca985f07b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:40,605 INFO [StoreOpener-f43b241499a6c8340b04792ca985f07b-1 {}] regionserver.HStore(327): Store=f43b241499a6c8340b04792ca985f07b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:40,606 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,607 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,607 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,608 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,608 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,610 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,613 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:40,614 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened f43b241499a6c8340b04792ca985f07b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75212428, jitterRate=0.12075251340866089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:40,614 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:50:40,614 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for f43b241499a6c8340b04792ca985f07b: Running coprocessor pre-open hook at 1731073840600Writing region info on filesystem at 1731073840600Initializing all the Stores at 1731073840602 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073840602Cleaning up temporary data from old regions at 1731073840608 (+6 ms)Running coprocessor post-open hooks at 1731073840614 (+6 ms)Region opened successfully at 1731073840614 2024-11-08T13:50:40,616 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b., pid=21, masterSystemTime=1731073840586 2024-11-08T13:50:40,618 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,618 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:50:40,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=f43b241499a6c8340b04792ca985f07b, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:40,622 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43b241499a6c8340b04792ca985f07b, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:40,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-08T13:50:40,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure f43b241499a6c8340b04792ca985f07b, server=6ccfea20c990,35805,1731073779264 in 191 msec 2024-11-08T13:50:40,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T13:50:40,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=f43b241499a6c8340b04792ca985f07b, ASSIGN in 356 msec 2024-11-08T13:50:40,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:40,629 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073840629"}]},"ts":"1731073840629"} 2024-11-08T13:50:40,632 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-08T13:50:40,633 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:40,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2170 sec 2024-11-08T13:50:41,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-08T13:50:41,999 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-08T13:50:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T13:50:42,452 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-08T13:50:42,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-08T13:50:42,453 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:42,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-08T13:50:42,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:42,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-08T13:50:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:42,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:42,477 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-08T13:50:42,481 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T13:50:42,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T13:50:42,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-08T13:50:42,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-08T13:50:45,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551656' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T13:50:45,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:53022 deadline: 1731073846493, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551656' does not exist 2024-11-08T13:50:45,497 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 6ccfea20c990,35805,1731073779264 for closing scanner id = 570217772525551656 for f43b241499a6c8340b04792ca985f07b of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551656' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:50:45,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=52 (was 57), ProcessCount=11 (was 11), AvailableMemoryMB=3340 (was 3343) 2024-11-08T13:50:45,511 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=52, ProcessCount=11, AvailableMemoryMB=3339 2024-11-08T13:50:45,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-08T13:50:45,516 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:45,516 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:45,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-08T13:50:45,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-08T13:50:45,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741847_1023 (size=411) 2024-11-08T13:50:45,935 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d957357f6086b6d19ff891d9d28e223a, NAME => 'testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:45,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741848_1024 (size=62) 2024-11-08T13:50:45,948 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:45,949 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing d957357f6086b6d19ff891d9d28e223a, disabling compactions & flushes 2024-11-08T13:50:45,949 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:45,949 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:45,949 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. after waiting 0 ms 2024-11-08T13:50:45,949 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:45,949 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:45,949 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for d957357f6086b6d19ff891d9d28e223a: Waiting for close lock at 1731073845949Disabling compacts and flushes for region at 1731073845949Disabling writes for close at 1731073845949Writing region close event to WAL at 1731073845949Closed at 1731073845949 2024-11-08T13:50:45,951 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:45,952 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731073845951"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073845951"}]},"ts":"1731073845951"} 2024-11-08T13:50:45,956 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:45,958 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:45,958 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073845958"}]},"ts":"1731073845958"} 2024-11-08T13:50:45,961 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-08T13:50:45,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d957357f6086b6d19ff891d9d28e223a, ASSIGN}] 2024-11-08T13:50:45,964 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d957357f6086b6d19ff891d9d28e223a, ASSIGN 2024-11-08T13:50:45,965 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d957357f6086b6d19ff891d9d28e223a, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-08T13:50:46,105 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:50:46,117 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d957357f6086b6d19ff891d9d28e223a, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:46,120 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d957357f6086b6d19ff891d9d28e223a, ASSIGN because future has completed 2024-11-08T13:50:46,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d957357f6086b6d19ff891d9d28e223a, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:46,281 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:46,281 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => d957357f6086b6d19ff891d9d28e223a, NAME => 'testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:46,282 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,282 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:46,282 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,282 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,285 INFO [StoreOpener-d957357f6086b6d19ff891d9d28e223a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,289 INFO [StoreOpener-d957357f6086b6d19ff891d9d28e223a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d957357f6086b6d19ff891d9d28e223a columnFamilyName testFamily 2024-11-08T13:50:46,289 DEBUG [StoreOpener-d957357f6086b6d19ff891d9d28e223a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:46,290 INFO [StoreOpener-d957357f6086b6d19ff891d9d28e223a-1 {}] regionserver.HStore(327): Store=d957357f6086b6d19ff891d9d28e223a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:46,291 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,293 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,294 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,296 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,296 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,299 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,302 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:46,303 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened d957357f6086b6d19ff891d9d28e223a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62404123, jitterRate=-0.07010610401630402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:46,303 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:50:46,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for d957357f6086b6d19ff891d9d28e223a: Running coprocessor pre-open hook at 1731073846282Writing region info on filesystem at 1731073846282Initializing all the Stores at 1731073846285 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073846285Cleaning up temporary data from old regions at 1731073846296 (+11 ms)Running coprocessor post-open hooks at 1731073846303 (+7 ms)Region opened successfully at 1731073846304 (+1 ms) 2024-11-08T13:50:46,306 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a., pid=24, masterSystemTime=1731073846273 2024-11-08T13:50:46,309 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:46,309 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:50:46,309 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d957357f6086b6d19ff891d9d28e223a, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:46,312 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d957357f6086b6d19ff891d9d28e223a, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:46,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-08T13:50:46,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure d957357f6086b6d19ff891d9d28e223a, server=6ccfea20c990,35805,1731073779264 in 193 msec 2024-11-08T13:50:46,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T13:50:46,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d957357f6086b6d19ff891d9d28e223a, ASSIGN in 354 msec 2024-11-08T13:50:46,320 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:46,320 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073846320"}]},"ts":"1731073846320"} 2024-11-08T13:50:46,323 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-08T13:50:46,324 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:46,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 811 msec 2024-11-08T13:50:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-08T13:50:46,533 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-08T13:50:46,533 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-08T13:50:46,534 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:46,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-08T13:50:46,546 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:46,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-08T13:50:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:46,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:46,558 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-08T13:50:46,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T13:50:46,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T13:50:46,564 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-08T13:50:46,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-08T13:50:48,002 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-08T13:50:49,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-08T13:50:49,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:49,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-08T13:50:49,025 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:49,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551660' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T13:50:49,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:53022 deadline: 1731073850573, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551660' does not exist 2024-11-08T13:50:49,576 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 6ccfea20c990,35805,1731073779264 for closing scanner id = 570217772525551660 for d957357f6086b6d19ff891d9d28e223a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551660' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:50:49,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1242505112_22 at /127.0.0.1:34464 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=48 (was 52), ProcessCount=11 (was 11), AvailableMemoryMB=3334 (was 3339) 2024-11-08T13:50:49,591 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=48, ProcessCount=11, AvailableMemoryMB=3334 2024-11-08T13:50:49,594 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:49,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-08T13:50:49,597 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:49,597 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:49,598 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-08T13:50:49,599 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T13:50:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741849_1025 (size=419) 2024-11-08T13:50:50,012 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => faa6294b5e929fdd22085d3dc7bf56da, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:50,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741850_1026 (size=70) 2024-11-08T13:50:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-08T13:50:50,429 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:50,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing faa6294b5e929fdd22085d3dc7bf56da, disabling compactions & flushes 2024-11-08T13:50:50,430 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,430 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. after waiting 0 ms 2024-11-08T13:50:50,431 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,431 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,431 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for faa6294b5e929fdd22085d3dc7bf56da: Waiting for close lock at 1731073850430Disabling compacts and flushes for region at 1731073850430Disabling writes for close at 1731073850431 (+1 ms)Writing region close event to WAL at 1731073850431Closed at 1731073850431 2024-11-08T13:50:50,435 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:50,436 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731073850435"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073850435"}]},"ts":"1731073850435"} 2024-11-08T13:50:50,440 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:50,441 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:50,442 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073850441"}]},"ts":"1731073850441"} 2024-11-08T13:50:50,444 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-08T13:50:50,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=faa6294b5e929fdd22085d3dc7bf56da, ASSIGN}] 2024-11-08T13:50:50,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=faa6294b5e929fdd22085d3dc7bf56da, ASSIGN 2024-11-08T13:50:50,447 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=faa6294b5e929fdd22085d3dc7bf56da, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:50,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=faa6294b5e929fdd22085d3dc7bf56da, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:50,605 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=faa6294b5e929fdd22085d3dc7bf56da, ASSIGN because future has completed 2024-11-08T13:50:50,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure faa6294b5e929fdd22085d3dc7bf56da, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T13:50:50,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,773 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => faa6294b5e929fdd22085d3dc7bf56da, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:50,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:50,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,777 INFO [StoreOpener-faa6294b5e929fdd22085d3dc7bf56da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,780 INFO [StoreOpener-faa6294b5e929fdd22085d3dc7bf56da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region faa6294b5e929fdd22085d3dc7bf56da columnFamilyName testFamily 2024-11-08T13:50:50,780 DEBUG [StoreOpener-faa6294b5e929fdd22085d3dc7bf56da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:50,781 INFO [StoreOpener-faa6294b5e929fdd22085d3dc7bf56da-1 {}] regionserver.HStore(327): Store=faa6294b5e929fdd22085d3dc7bf56da/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:50,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,782 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,784 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,784 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,786 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:50,791 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened faa6294b5e929fdd22085d3dc7bf56da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65924668, jitterRate=-0.01764589548110962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:50,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:50:50,792 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for faa6294b5e929fdd22085d3dc7bf56da: Running coprocessor pre-open hook at 1731073850775Writing region info on filesystem at 1731073850775Initializing all the Stores at 1731073850777 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073850777Cleaning up temporary data from old regions at 1731073850784 (+7 ms)Running coprocessor post-open hooks at 1731073850791 (+7 ms)Region opened successfully at 1731073850792 (+1 ms) 2024-11-08T13:50:50,793 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., pid=27, masterSystemTime=1731073850761 2024-11-08T13:50:50,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,796 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:50:50,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=faa6294b5e929fdd22085d3dc7bf56da, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:50,800 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure faa6294b5e929fdd22085d3dc7bf56da, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:50,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-08T13:50:50,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure faa6294b5e929fdd22085d3dc7bf56da, server=6ccfea20c990,35805,1731073779264 in 195 msec 2024-11-08T13:50:50,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T13:50:50,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=faa6294b5e929fdd22085d3dc7bf56da, ASSIGN in 360 msec 2024-11-08T13:50:50,808 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:50,808 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073850808"}]},"ts":"1731073850808"} 2024-11-08T13:50:50,811 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-08T13:50:50,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:50,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2190 sec 2024-11-08T13:50:51,787 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:50:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T13:50:52,631 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-08T13:50:52,631 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-08T13:50:52,632 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:52,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-08T13:50:52,637 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:50:52,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-08T13:50:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:50:52,651 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:50:52,651 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-08T13:50:52,654 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T13:50:52,655 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-08T13:50:53,484 INFO [master/6ccfea20c990:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T13:50:53,484 INFO [master/6ccfea20c990:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T13:50:54,006 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-08T13:50:58,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-08T13:50:58,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-08T13:50:58,657 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:50:58,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=40 (was 48), ProcessCount=11 (was 11), AvailableMemoryMB=3326 (was 3334) 2024-11-08T13:50:58,683 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=40, ProcessCount=11, AvailableMemoryMB=3326 2024-11-08T13:50:58,686 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:50:58,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-08T13:50:58,690 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:50:58,690 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:58,691 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-08T13:50:58,692 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:50:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T13:50:58,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741851_1027 (size=407) 2024-11-08T13:50:59,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-08T13:50:59,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-08T13:50:59,109 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c84533a4536eb5bec566e5fa5fabeb0b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:50:59,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741852_1028 (size=58) 2024-11-08T13:50:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-08T13:50:59,523 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:59,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing c84533a4536eb5bec566e5fa5fabeb0b, disabling compactions & flushes 2024-11-08T13:50:59,524 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,524 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. after waiting 0 ms 2024-11-08T13:50:59,525 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,525 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,525 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for c84533a4536eb5bec566e5fa5fabeb0b: Waiting for close lock at 1731073859524Disabling compacts and flushes for region at 1731073859524Disabling writes for close at 1731073859524Writing region close event to WAL at 1731073859525 (+1 ms)Closed at 1731073859525 2024-11-08T13:50:59,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:50:59,530 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731073859530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073859530"}]},"ts":"1731073859530"} 2024-11-08T13:50:59,534 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:50:59,536 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:50:59,536 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073859536"}]},"ts":"1731073859536"} 2024-11-08T13:50:59,540 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-08T13:50:59,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=c84533a4536eb5bec566e5fa5fabeb0b, ASSIGN}] 2024-11-08T13:50:59,542 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=c84533a4536eb5bec566e5fa5fabeb0b, ASSIGN 2024-11-08T13:50:59,543 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=c84533a4536eb5bec566e5fa5fabeb0b, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:50:59,694 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c84533a4536eb5bec566e5fa5fabeb0b, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:59,701 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=c84533a4536eb5bec566e5fa5fabeb0b, ASSIGN because future has completed 2024-11-08T13:50:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T13:50:59,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure c84533a4536eb5bec566e5fa5fabeb0b, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:50:59,861 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => c84533a4536eb5bec566e5fa5fabeb0b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:50:59,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:50:59,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,864 INFO [StoreOpener-c84533a4536eb5bec566e5fa5fabeb0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,866 INFO [StoreOpener-c84533a4536eb5bec566e5fa5fabeb0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c84533a4536eb5bec566e5fa5fabeb0b columnFamilyName testFamily 2024-11-08T13:50:59,866 DEBUG [StoreOpener-c84533a4536eb5bec566e5fa5fabeb0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:50:59,867 INFO [StoreOpener-c84533a4536eb5bec566e5fa5fabeb0b-1 {}] regionserver.HStore(327): Store=c84533a4536eb5bec566e5fa5fabeb0b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:50:59,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,871 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,873 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:50:59,874 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened c84533a4536eb5bec566e5fa5fabeb0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69825974, jitterRate=0.040488094091415405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:50:59,874 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:50:59,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for c84533a4536eb5bec566e5fa5fabeb0b: Running coprocessor pre-open hook at 1731073859862Writing region info on filesystem at 1731073859862Initializing all the Stores at 1731073859863 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073859863Cleaning up temporary data from old regions at 1731073859869 (+6 ms)Running coprocessor post-open hooks at 1731073859874 (+5 ms)Region opened successfully at 1731073859875 (+1 ms) 2024-11-08T13:50:59,876 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b., pid=30, masterSystemTime=1731073859856 2024-11-08T13:50:59,879 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,879 INFO [RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:50:59,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c84533a4536eb5bec566e5fa5fabeb0b, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:50:59,883 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure c84533a4536eb5bec566e5fa5fabeb0b, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:50:59,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-08T13:50:59,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure c84533a4536eb5bec566e5fa5fabeb0b, server=6ccfea20c990,35805,1731073779264 in 183 msec 2024-11-08T13:50:59,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T13:50:59,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=c84533a4536eb5bec566e5fa5fabeb0b, ASSIGN in 348 msec 2024-11-08T13:50:59,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:50:59,894 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073859893"}]},"ts":"1731073859893"} 2024-11-08T13:50:59,896 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-08T13:50:59,896 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:50:59,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2090 sec 2024-11-08T13:51:00,011 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-08T13:51:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T13:51:01,734 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-08T13:51:01,734 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-08T13:51:01,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:51:01,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-08T13:51:01,745 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:51:01,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-08T13:51:01,747 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd759451082065 with lease ID 0x859eea60bbeece40: from storage DS-080afe3e-d602-4fba-a664-88f8ab4add74 node DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138), blocks: 14, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T13:51:01,748 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd759451082065 with lease ID 0x859eea60bbeece40: from storage DS-1335aa46-62a4-47f5-aaca-0c9d04e5826c node DatanodeRegistration(127.0.0.1:42753, datanodeUuid=374d3115-1edb-4419-86e2-7e52c7018eb1, infoPort=36551, infoSecurePort=0, ipcPort=44495, storageInfo=lv=-57;cid=testClusterID;nsid=301446540;c=1731073776138), blocks: 14, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T13:51:01,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:51:01,760 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:51:01,760 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-08T13:51:01,763 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-08T13:51:01,765 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-08T13:51:01,766 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-08T13:51:01,766 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-08T13:51:05,393 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:51:07,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551668' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-08T13:51:07,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35805 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:53022 deadline: 1731073873773, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551668' does not exist 2024-11-08T13:51:07,777 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 6ccfea20c990,35805,1731073779264 for closing scanner id = 570217772525551668 for c84533a4536eb5bec566e5fa5fabeb0b of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '570217772525551668' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:51:07,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=34 (was 40), ProcessCount=11 (was 11), AvailableMemoryMB=3315 (was 3326) 2024-11-08T13:51:07,795 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=34, ProcessCount=11, AvailableMemoryMB=3314 2024-11-08T13:51:07,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:51:07,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-08T13:51:07,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:51:07,801 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:51:07,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-08T13:51:07,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:51:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T13:51:07,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741853_1029 (size=423) 2024-11-08T13:51:07,830 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:51:08,218 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d20764646fe5c76802ac4bc11f0903e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535 2024-11-08T13:51:08,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741854_1030 (size=74) 2024-11-08T13:51:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-08T13:51:08,632 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:51:08,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9d20764646fe5c76802ac4bc11f0903e, disabling compactions & flushes 2024-11-08T13:51:08,633 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. after waiting 0 ms 2024-11-08T13:51:08,633 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,634 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,634 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d20764646fe5c76802ac4bc11f0903e: Waiting for close lock at 1731073868633Disabling compacts and flushes for region at 1731073868633Disabling writes for close at 1731073868633Writing region close event to WAL at 1731073868634 (+1 ms)Closed at 1731073868634 2024-11-08T13:51:08,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:51:08,638 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731073868637"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731073868637"}]},"ts":"1731073868637"} 2024-11-08T13:51:08,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:51:08,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:51:08,644 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073868643"}]},"ts":"1731073868643"} 2024-11-08T13:51:08,647 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-08T13:51:08,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9d20764646fe5c76802ac4bc11f0903e, ASSIGN}] 2024-11-08T13:51:08,649 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9d20764646fe5c76802ac4bc11f0903e, ASSIGN 2024-11-08T13:51:08,650 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9d20764646fe5c76802ac4bc11f0903e, ASSIGN; state=OFFLINE, location=6ccfea20c990,35805,1731073779264; forceNewPlan=false, retain=false 2024-11-08T13:51:08,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9d20764646fe5c76802ac4bc11f0903e, regionState=OPENING, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:51:08,809 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9d20764646fe5c76802ac4bc11f0903e, ASSIGN because future has completed 2024-11-08T13:51:08,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d20764646fe5c76802ac4bc11f0903e, server=6ccfea20c990,35805,1731073779264}] 2024-11-08T13:51:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T13:51:08,976 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,976 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d20764646fe5c76802ac4bc11f0903e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:51:08,977 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,977 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:51:08,977 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,977 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,979 INFO [StoreOpener-9d20764646fe5c76802ac4bc11f0903e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,981 INFO [StoreOpener-9d20764646fe5c76802ac4bc11f0903e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d20764646fe5c76802ac4bc11f0903e columnFamilyName testFamily 2024-11-08T13:51:08,981 DEBUG [StoreOpener-9d20764646fe5c76802ac4bc11f0903e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:51:08,982 INFO [StoreOpener-9d20764646fe5c76802ac4bc11f0903e-1 {}] regionserver.HStore(327): Store=9d20764646fe5c76802ac4bc11f0903e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:51:08,982 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,983 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,983 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,984 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,984 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,986 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,988 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:51:08,989 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9d20764646fe5c76802ac4bc11f0903e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72047018, jitterRate=0.07358422875404358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:51:08,989 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:08,990 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9d20764646fe5c76802ac4bc11f0903e: Running coprocessor pre-open hook at 1731073868977Writing region info on filesystem at 1731073868977Initializing all the Stores at 1731073868979 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731073868979Cleaning up temporary data from old regions at 1731073868984 (+5 ms)Running coprocessor post-open hooks at 1731073868989 (+5 ms)Region opened successfully at 1731073868990 (+1 ms) 2024-11-08T13:51:08,992 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., pid=33, masterSystemTime=1731073868963 2024-11-08T13:51:08,994 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,994 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:08,995 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9d20764646fe5c76802ac4bc11f0903e, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35805,1731073779264 2024-11-08T13:51:08,998 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d20764646fe5c76802ac4bc11f0903e, server=6ccfea20c990,35805,1731073779264 because future has completed 2024-11-08T13:51:09,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T13:51:09,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d20764646fe5c76802ac4bc11f0903e, server=6ccfea20c990,35805,1731073779264 in 189 msec 2024-11-08T13:51:09,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T13:51:09,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9d20764646fe5c76802ac4bc11f0903e, ASSIGN in 355 msec 2024-11-08T13:51:09,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:51:09,006 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731073869006"}]},"ts":"1731073869006"} 2024-11-08T13:51:09,008 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-08T13:51:09,009 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:51:09,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2120 sec 2024-11-08T13:51:09,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-08T13:51:09,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-08T13:51:09,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-08T13:51:09,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-08T13:51:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43961 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T13:51:10,843 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-08T13:51:10,844 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-08T13:51:10,844 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:51:10,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-08T13:51:10,852 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:51:10,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-08T13:51:10,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:51:10,867 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-08T13:51:10,867 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-08T13:51:10,871 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-08T13:51:10,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-08T13:51:11,876 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-08T13:51:11,877 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-11-08T13:51:11,877 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:51:12,016 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-08T13:51:12,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-08T13:51:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2] 2024-11-08T13:51:12,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35805 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-08T13:51:12,926 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:51:13,876 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=6ccfea20c990:35805 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=978ms 2024-11-08T13:51:13,877 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=978ms 2024-11-08T13:51:13,877 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., hostname=6ccfea20c990,35805,1731073779264, seqNum=2 from cache 2024-11-08T13:51:13,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=47 (was 34) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3291 (was 3314) 2024-11-08T13:51:13,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:51:13,894 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:51:13,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T13:51:13,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T13:51:13,900 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:51:13,901 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:51:13,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T13:51:13,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T13:51:13,901 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=617532158, stopped=false 2024-11-08T13:51:13,901 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6ccfea20c990,43961,1731073778688 2024-11-08T13:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:51:13,904 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:51:13,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:51:13,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:51:13,904 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:51:13,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:51:13,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,905 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,35805,1731073779264' ***** 2024-11-08T13:51:13,905 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:51:13,905 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:51:13,905 INFO [RS:0;6ccfea20c990:35805 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:51:13,905 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for 7095e804fef2690ab6580c0941d14ab7 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for f43b241499a6c8340b04792ca985f07b 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for 7da3d795ef6a7df5fedbbfe4ac343af0 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for d957357f6086b6d19ff891d9d28e223a 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for b95944d9363e6561b6ea9b48f200d832 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:13,906 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for a3c9deaa9eec2c9ef9815dffeb3e01cc 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ea36ea2c345bf3b636ce12c5dbf3000 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d20764646fe5c76802ac4bc11f0903e 2024-11-08T13:51:13,907 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7095e804fef2690ab6580c0941d14ab7, disabling compactions & flushes 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:13,907 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,35805,1731073779264 2024-11-08T13:51:13,907 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:51:13,907 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. after waiting 0 ms 2024-11-08T13:51:13,907 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6ccfea20c990:35805. 2024-11-08T13:51:13,907 DEBUG [RS:0;6ccfea20c990:35805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:51:13,907 DEBUG [RS:0;6ccfea20c990:35805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:51:13,907 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T13:51:13,908 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-08T13:51:13,908 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:51:13,908 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:51:13,908 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:51:13,908 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7095e804fef2690ab6580c0941d14ab7, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, f43b241499a6c8340b04792ca985f07b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:13,908 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:51:13,908 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:51:13,910 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7095e804fef2690ab6580c0941d14ab7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:13,910 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-08T13:51:13,960 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/.tmp/testFamily/3be68772ddee41538db6cbb5c98bcd1d is 61, key is row-0/testFamily:testQualifier/1731073815925/Put/seqid=0 2024-11-08T13:51:13,967 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/info/467cd0e4cf9b41608ea6dfaefbce3f30 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7./info:regioninfo/1731073814064/Put/seqid=0 2024-11-08T13:51:13,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741855_1031 (size=5049) 2024-11-08T13:51:13,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741856_1032 (size=20564) 2024-11-08T13:51:13,996 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T13:51:13,997 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T13:51:14,109 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7095e804fef2690ab6580c0941d14ab7, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, f43b241499a6c8340b04792ca985f07b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:14,309 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7095e804fef2690ab6580c0941d14ab7, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, f43b241499a6c8340b04792ca985f07b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:14,372 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/.tmp/testFamily/3be68772ddee41538db6cbb5c98bcd1d 2024-11-08T13:51:14,375 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/info/467cd0e4cf9b41608ea6dfaefbce3f30 2024-11-08T13:51:14,395 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-08T13:51:14,419 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/.tmp/testFamily/3be68772ddee41538db6cbb5c98bcd1d as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/testFamily/3be68772ddee41538db6cbb5c98bcd1d 2024-11-08T13:51:14,428 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/testFamily/3be68772ddee41538db6cbb5c98bcd1d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:14,436 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7095e804fef2690ab6580c0941d14ab7 in 524ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:14,437 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/ns/d0eed3b1fdfb4d8a8bfb731b8a52c1a6 is 43, key is default/ns:d/1731073812541/Put/seqid=0 2024-11-08T13:51:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741857_1033 (size=5153) 2024-11-08T13:51:14,443 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextExceptionAsync/7095e804fef2690ab6580c0941d14ab7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:14,446 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:51:14,446 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7095e804fef2690ab6580c0941d14ab7: Waiting for close lock at 1731073873906Running coprocessor pre-close hooks at 1731073873907 (+1 ms)Disabling compacts and flushes for region at 1731073873907Disabling writes for close at 1731073873907Obtaining lock to block concurrent updates at 1731073873910 (+3 ms)Preparing flush snapshotting stores in 7095e804fef2690ab6580c0941d14ab7 at 1731073873910Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073873917 (+7 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. at 1731073873919 (+2 ms)Flushing 7095e804fef2690ab6580c0941d14ab7/testFamily: creating writer at 1731073873920 (+1 ms)Flushing 7095e804fef2690ab6580c0941d14ab7/testFamily: appending metadata at 1731073873955 (+35 ms)Flushing 7095e804fef2690ab6580c0941d14ab7/testFamily: closing flushed file at 1731073873957 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eaf3772: reopening flushed file at 1731073874417 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7095e804fef2690ab6580c0941d14ab7 in 524ms, sequenceid=8, compaction requested=false at 1731073874436 (+19 ms)Writing region close event to WAL at 1731073874438 (+2 ms)Running coprocessor post-close hooks at 1731073874444 (+6 ms)Closed at 1731073874446 (+2 ms) 2024-11-08T13:51:14,446 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731073812804.7095e804fef2690ab6580c0941d14ab7. 2024-11-08T13:51:14,447 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f43b241499a6c8340b04792ca985f07b, disabling compactions & flushes 2024-11-08T13:51:14,447 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:51:14,447 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:51:14,447 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. after waiting 0 ms 2024-11-08T13:51:14,447 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:51:14,447 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f43b241499a6c8340b04792ca985f07b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:14,468 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/.tmp/testFamily/acbcaf1900d24cd583f3446b79174061 is 61, key is row-0/testFamily:testQualifier/1731073842467/Put/seqid=0 2024-11-08T13:51:14,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741858_1034 (size=5049) 2024-11-08T13:51:14,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:51:14,510 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, f43b241499a6c8340b04792ca985f07b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:14,710 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, f43b241499a6c8340b04792ca985f07b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:14,845 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/ns/d0eed3b1fdfb4d8a8bfb731b8a52c1a6 2024-11-08T13:51:14,874 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/.tmp/testFamily/acbcaf1900d24cd583f3446b79174061 2024-11-08T13:51:14,877 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/table/954f0e308dc64fd7810a59e98ae0250f is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731073814082/Put/seqid=0 2024-11-08T13:51:14,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741859_1035 (size=6699) 2024-11-08T13:51:14,884 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/.tmp/testFamily/acbcaf1900d24cd583f3446b79174061 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/testFamily/acbcaf1900d24cd583f3446b79174061 2024-11-08T13:51:14,891 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/testFamily/acbcaf1900d24cd583f3446b79174061, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:14,894 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f43b241499a6c8340b04792ca985f07b in 447ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:14,899 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNextAsync/f43b241499a6c8340b04792ca985f07b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:14,900 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:51:14,900 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f43b241499a6c8340b04792ca985f07b: Waiting for close lock at 1731073874447Running coprocessor pre-close hooks at 1731073874447Disabling compacts and flushes for region at 1731073874447Disabling writes for close at 1731073874447Obtaining lock to block concurrent updates at 1731073874447Preparing flush snapshotting stores in f43b241499a6c8340b04792ca985f07b at 1731073874447Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073874447Flushing stores of testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. at 1731073874448 (+1 ms)Flushing f43b241499a6c8340b04792ca985f07b/testFamily: creating writer at 1731073874448Flushing f43b241499a6c8340b04792ca985f07b/testFamily: appending metadata at 1731073874468 (+20 ms)Flushing f43b241499a6c8340b04792ca985f07b/testFamily: closing flushed file at 1731073874468Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64b31577: reopening flushed file at 1731073874883 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f43b241499a6c8340b04792ca985f07b in 447ms, sequenceid=8, compaction requested=false at 1731073874894 (+11 ms)Writing region close event to WAL at 1731073874895 (+1 ms)Running coprocessor post-close hooks at 1731073874900 (+5 ms)Closed at 1731073874900 2024-11-08T13:51:14,900 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731073839415.f43b241499a6c8340b04792ca985f07b. 2024-11-08T13:51:14,900 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7da3d795ef6a7df5fedbbfe4ac343af0, disabling compactions & flushes 2024-11-08T13:51:14,900 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:51:14,901 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:51:14,901 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. after waiting 0 ms 2024-11-08T13:51:14,901 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:51:14,901 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7da3d795ef6a7df5fedbbfe4ac343af0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:14,910 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-08T13:51:14,910 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1325): Online Regions={7da3d795ef6a7df5fedbbfe4ac343af0=hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0., d957357f6086b6d19ff891d9d28e223a=testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a., b95944d9363e6561b6ea9b48f200d832=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., 1588230740=hbase:meta,,1.1588230740, faa6294b5e929fdd22085d3dc7bf56da=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., a3c9deaa9eec2c9ef9815dffeb3e01cc=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., 1ea36ea2c345bf3b636ce12c5dbf3000=testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., 9d20764646fe5c76802ac4bc11f0903e=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., c84533a4536eb5bec566e5fa5fabeb0b=hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.} 2024-11-08T13:51:14,911 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:14,917 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/.tmp/testFamily/bb6a6abef9324012865ec798249ba0e6 is 61, key is row-0/testFamily:testQualifier/1731073826160/Put/seqid=0 2024-11-08T13:51:14,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741860_1036 (size=5049) 2024-11-08T13:51:15,111 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:15,286 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/table/954f0e308dc64fd7810a59e98ae0250f 2024-11-08T13:51:15,299 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/info/467cd0e4cf9b41608ea6dfaefbce3f30 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/info/467cd0e4cf9b41608ea6dfaefbce3f30 2024-11-08T13:51:15,307 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/info/467cd0e4cf9b41608ea6dfaefbce3f30, entries=100, sequenceid=56, filesize=20.1 K 2024-11-08T13:51:15,308 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/ns/d0eed3b1fdfb4d8a8bfb731b8a52c1a6 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/ns/d0eed3b1fdfb4d8a8bfb731b8a52c1a6 2024-11-08T13:51:15,311 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ea36ea2c345bf3b636ce12c5dbf3000, 7da3d795ef6a7df5fedbbfe4ac343af0, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:15,316 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/ns/d0eed3b1fdfb4d8a8bfb731b8a52c1a6, entries=2, sequenceid=56, filesize=5.0 K 2024-11-08T13:51:15,318 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/.tmp/table/954f0e308dc64fd7810a59e98ae0250f as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/table/954f0e308dc64fd7810a59e98ae0250f 2024-11-08T13:51:15,323 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/.tmp/testFamily/bb6a6abef9324012865ec798249ba0e6 2024-11-08T13:51:15,327 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/table/954f0e308dc64fd7810a59e98ae0250f, entries=20, sequenceid=56, filesize=6.5 K 2024-11-08T13:51:15,328 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1420ms, sequenceid=56, compaction requested=false 2024-11-08T13:51:15,332 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/.tmp/testFamily/bb6a6abef9324012865ec798249ba0e6 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/testFamily/bb6a6abef9324012865ec798249ba0e6 2024-11-08T13:51:15,334 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-08T13:51:15,335 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:51:15,335 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:51:15,335 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731073873908Running coprocessor pre-close hooks at 1731073873908Disabling compacts and flushes for region at 1731073873908Disabling writes for close at 1731073873908Obtaining lock to block concurrent updates at 1731073873910 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1731073873910Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731073873917 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731073873919 (+2 ms)Flushing 1588230740/info: creating writer at 1731073873920 (+1 ms)Flushing 1588230740/info: appending metadata at 1731073873961 (+41 ms)Flushing 1588230740/info: closing flushed file at 1731073873962 (+1 ms)Flushing 1588230740/ns: creating writer at 1731073874417 (+455 ms)Flushing 1588230740/ns: appending metadata at 1731073874436 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731073874436Flushing 1588230740/table: creating writer at 1731073874860 (+424 ms)Flushing 1588230740/table: appending metadata at 1731073874877 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731073874877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68620053: reopening flushed file at 1731073875298 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14260f21: reopening flushed file at 1731073875307 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66c299fb: reopening flushed file at 1731073875317 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1420ms, sequenceid=56, compaction requested=false at 1731073875328 (+11 ms)Writing region close event to WAL at 1731073875330 (+2 ms)Running coprocessor post-close hooks at 1731073875334 (+4 ms)Closed at 1731073875335 (+1 ms) 2024-11-08T13:51:15,335 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T13:51:15,340 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/testFamily/bb6a6abef9324012865ec798249ba0e6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:15,342 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7da3d795ef6a7df5fedbbfe4ac343af0 in 441ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:15,347 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNextAsync/7da3d795ef6a7df5fedbbfe4ac343af0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:15,347 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:51:15,347 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7da3d795ef6a7df5fedbbfe4ac343af0: Waiting for close lock at 1731073874900Running coprocessor pre-close hooks at 1731073874900Disabling compacts and flushes for region at 1731073874900Disabling writes for close at 1731073874901 (+1 ms)Obtaining lock to block concurrent updates at 1731073874901Preparing flush snapshotting stores in 7da3d795ef6a7df5fedbbfe4ac343af0 at 1731073874901Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073874901Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. at 1731073874902 (+1 ms)Flushing 7da3d795ef6a7df5fedbbfe4ac343af0/testFamily: creating writer at 1731073874902Flushing 7da3d795ef6a7df5fedbbfe4ac343af0/testFamily: appending metadata at 1731073874917 (+15 ms)Flushing 7da3d795ef6a7df5fedbbfe4ac343af0/testFamily: closing flushed file at 1731073874917Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dfa38a9: reopening flushed file at 1731073875331 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7da3d795ef6a7df5fedbbfe4ac343af0 in 441ms, sequenceid=8, compaction requested=false at 1731073875342 (+11 ms)Writing region close event to WAL at 1731073875342Running coprocessor post-close hooks at 1731073875347 (+5 ms)Closed at 1731073875347 2024-11-08T13:51:15,348 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731073823121.7da3d795ef6a7df5fedbbfe4ac343af0. 2024-11-08T13:51:15,348 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d957357f6086b6d19ff891d9d28e223a, disabling compactions & flushes 2024-11-08T13:51:15,348 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:51:15,348 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:51:15,348 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. after waiting 0 ms 2024-11-08T13:51:15,348 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:51:15,348 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d957357f6086b6d19ff891d9d28e223a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:15,363 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/.tmp/testFamily/8f39c9d9799c4709a9d68d968dce8818 is 61, key is row-0/testFamily:testQualifier/1731073846550/Put/seqid=0 2024-11-08T13:51:15,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741861_1037 (size=5049) 2024-11-08T13:51:15,512 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:15,712 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, d957357f6086b6d19ff891d9d28e223a, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:15,771 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/.tmp/testFamily/8f39c9d9799c4709a9d68d968dce8818 2024-11-08T13:51:15,780 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=6ccfea20c990:43961 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-08T13:51:15,786 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/.tmp/testFamily/8f39c9d9799c4709a9d68d968dce8818 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/testFamily/8f39c9d9799c4709a9d68d968dce8818 2024-11-08T13:51:15,792 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/testFamily/8f39c9d9799c4709a9d68d968dce8818, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:15,793 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d957357f6086b6d19ff891d9d28e223a in 445ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:15,798 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnNext/d957357f6086b6d19ff891d9d28e223a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:15,799 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d957357f6086b6d19ff891d9d28e223a: Waiting for close lock at 1731073875348Running coprocessor pre-close hooks at 1731073875348Disabling compacts and flushes for region at 1731073875348Disabling writes for close at 1731073875348Obtaining lock to block concurrent updates at 1731073875348Preparing flush snapshotting stores in d957357f6086b6d19ff891d9d28e223a at 1731073875348Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073875348Flushing stores of testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. at 1731073875349 (+1 ms)Flushing d957357f6086b6d19ff891d9d28e223a/testFamily: creating writer at 1731073875349Flushing d957357f6086b6d19ff891d9d28e223a/testFamily: appending metadata at 1731073875363 (+14 ms)Flushing d957357f6086b6d19ff891d9d28e223a/testFamily: closing flushed file at 1731073875363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1f93d4: reopening flushed file at 1731073875784 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d957357f6086b6d19ff891d9d28e223a in 445ms, sequenceid=8, compaction requested=false at 1731073875794 (+10 ms)Writing region close event to WAL at 1731073875794Running coprocessor post-close hooks at 1731073875799 (+5 ms)Closed at 1731073875799 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731073845513.d957357f6086b6d19ff891d9d28e223a. 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b95944d9363e6561b6ea9b48f200d832, disabling compactions & flushes 2024-11-08T13:51:15,799 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. after waiting 0 ms 2024-11-08T13:51:15,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:51:15,800 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b95944d9363e6561b6ea9b48f200d832 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:15,815 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/.tmp/testFamily/aabd5b2669924eb4b2e07228e65ca403 is 61, key is row-0/testFamily:testQualifier/1731073836370/Put/seqid=0 2024-11-08T13:51:15,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741862_1038 (size=5049) 2024-11-08T13:51:15,912 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-08T13:51:15,912 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1325): Online Regions={b95944d9363e6561b6ea9b48f200d832=testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., faa6294b5e929fdd22085d3dc7bf56da=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., a3c9deaa9eec2c9ef9815dffeb3e01cc=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., 1ea36ea2c345bf3b636ce12c5dbf3000=testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., 9d20764646fe5c76802ac4bc11f0903e=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., c84533a4536eb5bec566e5fa5fabeb0b=hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.} 2024-11-08T13:51:15,912 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:16,113 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, b95944d9363e6561b6ea9b48f200d832, c84533a4536eb5bec566e5fa5fabeb0b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:16,223 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/.tmp/testFamily/aabd5b2669924eb4b2e07228e65ca403 2024-11-08T13:51:16,237 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/.tmp/testFamily/aabd5b2669924eb4b2e07228e65ca403 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/testFamily/aabd5b2669924eb4b2e07228e65ca403 2024-11-08T13:51:16,244 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/testFamily/aabd5b2669924eb4b2e07228e65ca403, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:16,245 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b95944d9363e6561b6ea9b48f200d832 in 445ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:16,250 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScanner/b95944d9363e6561b6ea9b48f200d832/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:16,250 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:51:16,250 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b95944d9363e6561b6ea9b48f200d832: Waiting for close lock at 1731073875799Running coprocessor pre-close hooks at 1731073875799Disabling compacts and flushes for region at 1731073875799Disabling writes for close at 1731073875799Obtaining lock to block concurrent updates at 1731073875800 (+1 ms)Preparing flush snapshotting stores in b95944d9363e6561b6ea9b48f200d832 at 1731073875800Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073875800Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. at 1731073875800Flushing b95944d9363e6561b6ea9b48f200d832/testFamily: creating writer at 1731073875801 (+1 ms)Flushing b95944d9363e6561b6ea9b48f200d832/testFamily: appending metadata at 1731073875815 (+14 ms)Flushing b95944d9363e6561b6ea9b48f200d832/testFamily: closing flushed file at 1731073875815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b01d365: reopening flushed file at 1731073876236 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b95944d9363e6561b6ea9b48f200d832 in 445ms, sequenceid=8, compaction requested=false at 1731073876245 (+9 ms)Writing region close event to WAL at 1731073876246 (+1 ms)Running coprocessor post-close hooks at 1731073876250 (+4 ms)Closed at 1731073876250 2024-11-08T13:51:16,251 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731073835331.b95944d9363e6561b6ea9b48f200d832. 2024-11-08T13:51:16,251 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing faa6294b5e929fdd22085d3dc7bf56da, disabling compactions & flushes 2024-11-08T13:51:16,251 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:51:16,251 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:51:16,251 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. after waiting 0 ms 2024-11-08T13:51:16,251 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:51:16,251 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing faa6294b5e929fdd22085d3dc7bf56da 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:16,266 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/.tmp/testFamily/6b884de071df40439fea5c70083cdeab is 61, key is row-0/testFamily:testQualifier/1731073852640/Put/seqid=0 2024-11-08T13:51:16,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741863_1039 (size=5049) 2024-11-08T13:51:16,313 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, c84533a4536eb5bec566e5fa5fabeb0b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:16,513 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, c84533a4536eb5bec566e5fa5fabeb0b, faa6294b5e929fdd22085d3dc7bf56da 2024-11-08T13:51:16,674 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/.tmp/testFamily/6b884de071df40439fea5c70083cdeab 2024-11-08T13:51:16,688 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/.tmp/testFamily/6b884de071df40439fea5c70083cdeab as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/testFamily/6b884de071df40439fea5c70083cdeab 2024-11-08T13:51:16,696 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/testFamily/6b884de071df40439fea5c70083cdeab, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:16,697 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for faa6294b5e929fdd22085d3dc7bf56da in 446ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:16,702 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/faa6294b5e929fdd22085d3dc7bf56da/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:16,702 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for faa6294b5e929fdd22085d3dc7bf56da: Waiting for close lock at 1731073876251Running coprocessor pre-close hooks at 1731073876251Disabling compacts and flushes for region at 1731073876251Disabling writes for close at 1731073876251Obtaining lock to block concurrent updates at 1731073876251Preparing flush snapshotting stores in faa6294b5e929fdd22085d3dc7bf56da at 1731073876251Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073876251Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. at 1731073876252 (+1 ms)Flushing faa6294b5e929fdd22085d3dc7bf56da/testFamily: creating writer at 1731073876252Flushing faa6294b5e929fdd22085d3dc7bf56da/testFamily: appending metadata at 1731073876266 (+14 ms)Flushing faa6294b5e929fdd22085d3dc7bf56da/testFamily: closing flushed file at 1731073876266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b724913: reopening flushed file at 1731073876687 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for faa6294b5e929fdd22085d3dc7bf56da in 446ms, sequenceid=8, compaction requested=false at 1731073876697 (+10 ms)Writing region close event to WAL at 1731073876698 (+1 ms)Running coprocessor post-close hooks at 1731073876702 (+4 ms)Closed at 1731073876702 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731073849594.faa6294b5e929fdd22085d3dc7bf56da. 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a3c9deaa9eec2c9ef9815dffeb3e01cc, disabling compactions & flushes 2024-11-08T13:51:16,703 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. after waiting 0 ms 2024-11-08T13:51:16,703 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:51:16,703 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a3c9deaa9eec2c9ef9815dffeb3e01cc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:16,714 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:16,718 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/.tmp/testFamily/70c34dd1eb7f4a719fa9202b6f3f1f89 is 61, key is row-0/testFamily:testQualifier/1731073817067/Put/seqid=0 2024-11-08T13:51:16,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741864_1040 (size=5049) 2024-11-08T13:51:16,914 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-08T13:51:16,914 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1325): Online Regions={a3c9deaa9eec2c9ef9815dffeb3e01cc=hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., 1ea36ea2c345bf3b636ce12c5dbf3000=testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., 9d20764646fe5c76802ac4bc11f0903e=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., c84533a4536eb5bec566e5fa5fabeb0b=hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.} 2024-11-08T13:51:16,915 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:17,115 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, a3c9deaa9eec2c9ef9815dffeb3e01cc, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:17,125 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/.tmp/testFamily/70c34dd1eb7f4a719fa9202b6f3f1f89 2024-11-08T13:51:17,140 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/.tmp/testFamily/70c34dd1eb7f4a719fa9202b6f3f1f89 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily/70c34dd1eb7f4a719fa9202b6f3f1f89 2024-11-08T13:51:17,148 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily/70c34dd1eb7f4a719fa9202b6f3f1f89, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:17,149 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a3c9deaa9eec2c9ef9815dffeb3e01cc in 446ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:17,154 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnOpenScanner/a3c9deaa9eec2c9ef9815dffeb3e01cc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:17,155 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a3c9deaa9eec2c9ef9815dffeb3e01cc: Waiting for close lock at 1731073876703Running coprocessor pre-close hooks at 1731073876703Disabling compacts and flushes for region at 1731073876703Disabling writes for close at 1731073876703Obtaining lock to block concurrent updates at 1731073876703Preparing flush snapshotting stores in a3c9deaa9eec2c9ef9815dffeb3e01cc at 1731073876703Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073876703Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. at 1731073876704 (+1 ms)Flushing a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily: creating writer at 1731073876704Flushing a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily: appending metadata at 1731073876717 (+13 ms)Flushing a3c9deaa9eec2c9ef9815dffeb3e01cc/testFamily: closing flushed file at 1731073876718 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fd513bc: reopening flushed file at 1731073877138 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a3c9deaa9eec2c9ef9815dffeb3e01cc in 446ms, sequenceid=8, compaction requested=false at 1731073877149 (+11 ms)Writing region close event to WAL at 1731073877150 (+1 ms)Running coprocessor post-close hooks at 1731073877154 (+4 ms)Closed at 1731073877154 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731073816024.a3c9deaa9eec2c9ef9815dffeb3e01cc. 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ea36ea2c345bf3b636ce12c5dbf3000, disabling compactions & flushes 2024-11-08T13:51:17,155 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. after waiting 0 ms 2024-11-08T13:51:17,155 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:51:17,155 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1ea36ea2c345bf3b636ce12c5dbf3000 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:17,171 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/.tmp/testFamily/a629097253224936809bbcd7719a1a49 is 61, key is row-0/testFamily:testQualifier/1731073835269/Put/seqid=0 2024-11-08T13:51:17,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741865_1041 (size=5049) 2024-11-08T13:51:17,316 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:17,516 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 1ea36ea2c345bf3b636ce12c5dbf3000, 9d20764646fe5c76802ac4bc11f0903e, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:17,576 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/.tmp/testFamily/a629097253224936809bbcd7719a1a49 2024-11-08T13:51:17,589 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/.tmp/testFamily/a629097253224936809bbcd7719a1a49 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/testFamily/a629097253224936809bbcd7719a1a49 2024-11-08T13:51:17,596 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/testFamily/a629097253224936809bbcd7719a1a49, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:17,597 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ea36ea2c345bf3b636ce12c5dbf3000 in 442ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:17,602 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testRetryOutOfOrderScannerNextException/1ea36ea2c345bf3b636ce12c5dbf3000/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:17,603 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:51:17,603 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ea36ea2c345bf3b636ce12c5dbf3000: Waiting for close lock at 1731073877155Running coprocessor pre-close hooks at 1731073877155Disabling compacts and flushes for region at 1731073877155Disabling writes for close at 1731073877155Obtaining lock to block concurrent updates at 1731073877155Preparing flush snapshotting stores in 1ea36ea2c345bf3b636ce12c5dbf3000 at 1731073877155Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073877155Flushing stores of testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. at 1731073877156 (+1 ms)Flushing 1ea36ea2c345bf3b636ce12c5dbf3000/testFamily: creating writer at 1731073877156Flushing 1ea36ea2c345bf3b636ce12c5dbf3000/testFamily: appending metadata at 1731073877171 (+15 ms)Flushing 1ea36ea2c345bf3b636ce12c5dbf3000/testFamily: closing flushed file at 1731073877171Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b7343f: reopening flushed file at 1731073877588 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ea36ea2c345bf3b636ce12c5dbf3000 in 442ms, sequenceid=8, compaction requested=false at 1731073877597 (+9 ms)Writing region close event to WAL at 1731073877598 (+1 ms)Running coprocessor post-close hooks at 1731073877603 (+5 ms)Closed at 1731073877603 2024-11-08T13:51:17,603 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731073832213.1ea36ea2c345bf3b636ce12c5dbf3000. 2024-11-08T13:51:17,604 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d20764646fe5c76802ac4bc11f0903e, disabling compactions & flushes 2024-11-08T13:51:17,604 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:17,604 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:17,604 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. after waiting 0 ms 2024-11-08T13:51:17,604 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:17,604 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9d20764646fe5c76802ac4bc11f0903e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:17,626 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/.tmp/testFamily/388cdd43927f4f739be059f100e371e7 is 61, key is row-0/testFamily:testQualifier/1731073870857/Put/seqid=0 2024-11-08T13:51:17,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741866_1042 (size=5049) 2024-11-08T13:51:17,716 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 9d20764646fe5c76802ac4bc11f0903e, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:17,917 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T13:51:17,917 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1325): Online Regions={9d20764646fe5c76802ac4bc11f0903e=testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., c84533a4536eb5bec566e5fa5fabeb0b=hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b.} 2024-11-08T13:51:17,917 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on 9d20764646fe5c76802ac4bc11f0903e, c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:18,032 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/.tmp/testFamily/388cdd43927f4f739be059f100e371e7 2024-11-08T13:51:18,041 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/.tmp/testFamily/388cdd43927f4f739be059f100e371e7 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/testFamily/388cdd43927f4f739be059f100e371e7 2024-11-08T13:51:18,048 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/testFamily/388cdd43927f4f739be059f100e371e7, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:18,049 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9d20764646fe5c76802ac4bc11f0903e in 445ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:18,053 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/default/testNormalScanTimeoutOnOpenScannerAsync/9d20764646fe5c76802ac4bc11f0903e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:18,054 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d20764646fe5c76802ac4bc11f0903e: Waiting for close lock at 1731073877604Running coprocessor pre-close hooks at 1731073877604Disabling compacts and flushes for region at 1731073877604Disabling writes for close at 1731073877604Obtaining lock to block concurrent updates at 1731073877604Preparing flush snapshotting stores in 9d20764646fe5c76802ac4bc11f0903e at 1731073877604Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073877604Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. at 1731073877605 (+1 ms)Flushing 9d20764646fe5c76802ac4bc11f0903e/testFamily: creating writer at 1731073877605Flushing 9d20764646fe5c76802ac4bc11f0903e/testFamily: appending metadata at 1731073877625 (+20 ms)Flushing 9d20764646fe5c76802ac4bc11f0903e/testFamily: closing flushed file at 1731073877626 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a015df: reopening flushed file at 1731073878039 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9d20764646fe5c76802ac4bc11f0903e in 445ms, sequenceid=8, compaction requested=false at 1731073878049 (+10 ms)Writing region close event to WAL at 1731073878049Running coprocessor post-close hooks at 1731073878054 (+5 ms)Closed at 1731073878054 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731073867797.9d20764646fe5c76802ac4bc11f0903e. 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c84533a4536eb5bec566e5fa5fabeb0b, disabling compactions & flushes 2024-11-08T13:51:18,054 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. after waiting 0 ms 2024-11-08T13:51:18,054 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:51:18,054 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c84533a4536eb5bec566e5fa5fabeb0b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-08T13:51:18,077 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/.tmp/testFamily/9691b2758ccc4c4a88bb467da88201a6 is 61, key is row-0/testFamily:testQualifier/1731073861751/Put/seqid=0 2024-11-08T13:51:18,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741867_1043 (size=5049) 2024-11-08T13:51:18,117 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:18,318 DEBUG [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1351): Waiting on c84533a4536eb5bec566e5fa5fabeb0b 2024-11-08T13:51:18,484 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/.tmp/testFamily/9691b2758ccc4c4a88bb467da88201a6 2024-11-08T13:51:18,498 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/.tmp/testFamily/9691b2758ccc4c4a88bb467da88201a6 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/testFamily/9691b2758ccc4c4a88bb467da88201a6 2024-11-08T13:51:18,504 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/testFamily/9691b2758ccc4c4a88bb467da88201a6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-08T13:51:18,505 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c84533a4536eb5bec566e5fa5fabeb0b in 451ms, sequenceid=8, compaction requested=false 2024-11-08T13:51:18,511 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/data/hbase/testMetaScanTimeoutOnNext/c84533a4536eb5bec566e5fa5fabeb0b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-08T13:51:18,511 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:51:18,511 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c84533a4536eb5bec566e5fa5fabeb0b: Waiting for close lock at 1731073878054Running coprocessor pre-close hooks at 1731073878054Disabling compacts and flushes for region at 1731073878054Disabling writes for close at 1731073878054Obtaining lock to block concurrent updates at 1731073878054Preparing flush snapshotting stores in c84533a4536eb5bec566e5fa5fabeb0b at 1731073878054Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731073878055 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. at 1731073878055Flushing c84533a4536eb5bec566e5fa5fabeb0b/testFamily: creating writer at 1731073878055Flushing c84533a4536eb5bec566e5fa5fabeb0b/testFamily: appending metadata at 1731073878076 (+21 ms)Flushing c84533a4536eb5bec566e5fa5fabeb0b/testFamily: closing flushed file at 1731073878077 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46d35ae0: reopening flushed file at 1731073878497 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c84533a4536eb5bec566e5fa5fabeb0b in 451ms, sequenceid=8, compaction requested=false at 1731073878505 (+8 ms)Writing region close event to WAL at 1731073878506 (+1 ms)Running coprocessor post-close hooks at 1731073878511 (+5 ms)Closed at 1731073878511 2024-11-08T13:51:18,512 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731073858686.c84533a4536eb5bec566e5fa5fabeb0b. 2024-11-08T13:51:18,518 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,35805,1731073779264; all regions closed. 2024-11-08T13:51:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741834_1010 (size=19479) 2024-11-08T13:51:18,529 DEBUG [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/oldWALs 2024-11-08T13:51:18,529 INFO [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35805%2C1731073779264.meta:.meta(num 1731073812384) 2024-11-08T13:51:18,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741833_1009 (size=20031) 2024-11-08T13:51:18,532 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/WALs/6ccfea20c990,35805,1731073779264/6ccfea20c990%2C35805%2C1731073779264.1731073811992 not finished, retry = 0 2024-11-08T13:51:18,638 DEBUG [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/oldWALs 2024-11-08T13:51:18,638 INFO [RS:0;6ccfea20c990:35805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35805%2C1731073779264:(num 1731073811992) 2024-11-08T13:51:18,638 DEBUG [RS:0;6ccfea20c990:35805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:51:18,638 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:51:18,639 INFO [RS:0;6ccfea20c990:35805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:51:18,639 INFO [RS:0;6ccfea20c990:35805 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T13:51:18,640 INFO [RS:0;6ccfea20c990:35805 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:51:18,640 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:51:18,642 INFO [RS:0;6ccfea20c990:35805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35805 2024-11-08T13:51:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,35805,1731073779264 2024-11-08T13:51:18,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:51:18,647 INFO [RS:0;6ccfea20c990:35805 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:51:18,647 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,35805,1731073779264] 2024-11-08T13:51:18,648 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,35805,1731073779264 already deleted, retry=false 2024-11-08T13:51:18,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,35805,1731073779264 expired; onlineServers=0 2024-11-08T13:51:18,649 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6ccfea20c990,43961,1731073778688' ***** 2024-11-08T13:51:18,649 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T13:51:18,649 INFO [M:0;6ccfea20c990:43961 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:51:18,649 INFO [M:0;6ccfea20c990:43961 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:51:18,649 DEBUG [M:0;6ccfea20c990:43961 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T13:51:18,649 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T13:51:18,649 DEBUG [M:0;6ccfea20c990:43961 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T13:51:18,649 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731073781816 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731073781816,5,FailOnTimeoutGroup] 2024-11-08T13:51:18,649 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731073781817 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731073781817,5,FailOnTimeoutGroup] 2024-11-08T13:51:18,649 INFO [M:0;6ccfea20c990:43961 {}] hbase.ChoreService(370): Chore service for: master/6ccfea20c990:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T13:51:18,649 INFO [M:0;6ccfea20c990:43961 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:51:18,650 DEBUG [M:0;6ccfea20c990:43961 {}] master.HMaster(1795): Stopping service threads 2024-11-08T13:51:18,650 INFO [M:0;6ccfea20c990:43961 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T13:51:18,650 INFO [M:0;6ccfea20c990:43961 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:51:18,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T13:51:18,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:51:18,650 INFO [M:0;6ccfea20c990:43961 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:51:18,650 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T13:51:18,651 DEBUG [M:0;6ccfea20c990:43961 {}] zookeeper.ZKUtil(347): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T13:51:18,651 WARN [M:0;6ccfea20c990:43961 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T13:51:18,652 INFO [M:0;6ccfea20c990:43961 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/.lastflushedseqids 2024-11-08T13:51:18,660 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:51:18,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741868_1044 (size=712) 2024-11-08T13:51:18,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:51:18,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35805-0x1001f98fd470001, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:51:18,750 INFO [RS:0;6ccfea20c990:35805 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:51:18,750 INFO [RS:0;6ccfea20c990:35805 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,35805,1731073779264; zookeeper connection closed. 2024-11-08T13:51:18,751 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@20354ce2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@20354ce2 2024-11-08T13:51:18,752 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T13:51:19,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T13:51:19,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-08T13:51:19,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-08T13:51:19,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-08T13:51:19,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-08T13:51:19,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-08T13:51:19,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:51:19,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-08T13:51:19,026 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-08T13:51:19,027 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-08T13:51:19,027 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-08T13:51:19,027 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-08T13:51:19,063 INFO [M:0;6ccfea20c990:43961 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:51:19,064 INFO [M:0;6ccfea20c990:43961 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T13:51:19,064 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:51:19,065 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:51:19,065 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:51:19,065 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:51:19,065 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:51:19,066 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.07 KB heapSize=191.13 KB 2024-11-08T13:51:19,088 DEBUG [M:0;6ccfea20c990:43961 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60be892eccf049d4bd201afe42e4cb0a is 82, key is hbase:meta,,1/info:regioninfo/1731073812472/Put/seqid=0 2024-11-08T13:51:19,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741869_1045 (size=5672) 2024-11-08T13:51:19,495 INFO [M:0;6ccfea20c990:43961 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60be892eccf049d4bd201afe42e4cb0a 2024-11-08T13:51:19,527 DEBUG [M:0;6ccfea20c990:43961 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e780b70415e4e758aced40f372621e7 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731073814089/Put/seqid=0 2024-11-08T13:51:19,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741870_1046 (size=14068) 2024-11-08T13:51:19,934 INFO [M:0;6ccfea20c990:43961 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.51 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e780b70415e4e758aced40f372621e7 2024-11-08T13:51:19,949 INFO [M:0;6ccfea20c990:43961 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5e780b70415e4e758aced40f372621e7 2024-11-08T13:51:19,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:51:19,966 DEBUG [M:0;6ccfea20c990:43961 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6faf63794834f6a8260dc3c4a50106a is 69, key is 6ccfea20c990,35805,1731073779264/rs:state/1731073781848/Put/seqid=0 2024-11-08T13:51:19,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741871_1047 (size=5156) 2024-11-08T13:51:20,373 INFO [M:0;6ccfea20c990:43961 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6faf63794834f6a8260dc3c4a50106a 2024-11-08T13:51:20,388 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/60be892eccf049d4bd201afe42e4cb0a as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/60be892eccf049d4bd201afe42e4cb0a 2024-11-08T13:51:20,396 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/60be892eccf049d4bd201afe42e4cb0a, entries=8, sequenceid=319, filesize=5.5 K 2024-11-08T13:51:20,398 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e780b70415e4e758aced40f372621e7 as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5e780b70415e4e758aced40f372621e7 2024-11-08T13:51:20,404 INFO [M:0;6ccfea20c990:43961 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5e780b70415e4e758aced40f372621e7 2024-11-08T13:51:20,405 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5e780b70415e4e758aced40f372621e7, entries=33, sequenceid=319, filesize=13.7 K 2024-11-08T13:51:20,406 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d6faf63794834f6a8260dc3c4a50106a as hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6faf63794834f6a8260dc3c4a50106a 2024-11-08T13:51:20,412 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44643/user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d6faf63794834f6a8260dc3c4a50106a, entries=1, sequenceid=319, filesize=5.0 K 2024-11-08T13:51:20,413 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.07 KB/166979, heapSize ~190.84 KB/195416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1348ms, sequenceid=319, compaction requested=false 2024-11-08T13:51:20,414 INFO [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:51:20,414 DEBUG [M:0;6ccfea20c990:43961 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731073879064Disabling compacts and flushes for region at 1731073879064Disabling writes for close at 1731073879065 (+1 ms)Obtaining lock to block concurrent updates at 1731073879066 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731073879066Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166979, getHeapSize=195656, getOffHeapSize=0, getCellsCount=366 at 1731073879067 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731073879068 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731073879068Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731073879087 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731073879087Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731073879509 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731073879526 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731073879526Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731073879949 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731073879965 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731073879965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47275ee3: reopening flushed file at 1731073880386 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23a733ca: reopening flushed file at 1731073880397 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0e18b6: reopening flushed file at 1731073880405 (+8 ms)Finished flush of dataSize ~163.07 KB/166979, heapSize ~190.84 KB/195416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1348ms, sequenceid=319, compaction requested=false at 1731073880413 (+8 ms)Writing region close event to WAL at 1731073880414 (+1 ms)Closed at 1731073880414 2024-11-08T13:51:20,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42753 is added to blk_1073741830_1006 (size=188321) 2024-11-08T13:51:20,417 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f7f38374-a1eb-77dd-a055-113be02b5535/MasterData/WALs/6ccfea20c990,43961,1731073778688/6ccfea20c990%2C43961%2C1731073778688.1731073781327 not finished, retry = 0 2024-11-08T13:51:20,519 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:51:20,519 INFO [M:0;6ccfea20c990:43961 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T13:51:20,519 INFO [M:0;6ccfea20c990:43961 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43961 2024-11-08T13:51:20,519 INFO [M:0;6ccfea20c990:43961 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:51:20,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:51:20,621 INFO [M:0;6ccfea20c990:43961 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:51:20,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43961-0x1001f98fd470000, quorum=127.0.0.1:50109, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:51:20,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@341cfc82{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:51:20,631 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7061d16d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:51:20,631 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:51:20,632 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47169f40{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:51:20,632 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72127eee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.log.dir/,STOPPED} 2024-11-08T13:51:20,635 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T13:51:20,635 WARN [BP-593724743-172.17.0.2-1731073776138 heartbeating to localhost/127.0.0.1:44643 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T13:51:20,635 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T13:51:20,635 WARN [BP-593724743-172.17.0.2-1731073776138 heartbeating to localhost/127.0.0.1:44643 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-593724743-172.17.0.2-1731073776138 (Datanode Uuid 374d3115-1edb-4419-86e2-7e52c7018eb1) service to localhost/127.0.0.1:44643 2024-11-08T13:51:20,636 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/data/data1/current/BP-593724743-172.17.0.2-1731073776138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:51:20,637 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/cluster_6e8532f8-df74-6fdb-eee4-e34ed34c66f5/data/data2/current/BP-593724743-172.17.0.2-1731073776138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:51:20,637 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T13:51:20,644 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T13:51:20,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:51:20,645 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:51:20,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:51:20,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1a9ebeb3-ed30-441f-2010-7f36cac49188/hadoop.log.dir/,STOPPED} 2024-11-08T13:51:20,653 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T13:51:20,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down