2024-12-08 21:02:22,473 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-08 21:02:22,510 main DEBUG Took 0.033326 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 21:02:22,510 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 21:02:22,511 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 21:02:22,512 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 21:02:22,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,527 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 21:02:22,546 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,549 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,550 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,552 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,553 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,554 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,558 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,559 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,559 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,560 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,561 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:02:22,562 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,562 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 21:02:22,564 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:02:22,566 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 21:02:22,569 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 21:02:22,569 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 21:02:22,571 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 21:02:22,572 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 21:02:22,582 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 21:02:22,586 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 21:02:22,588 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 21:02:22,589 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 21:02:22,590 main DEBUG createAppenders(={Console}) 2024-12-08 21:02:22,591 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-08 21:02:22,592 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-08 21:02:22,592 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-08 21:02:22,593 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 21:02:22,593 main DEBUG OutputStream closed 2024-12-08 21:02:22,594 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 21:02:22,595 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 21:02:22,596 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-08 21:02:22,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 21:02:22,691 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 21:02:22,693 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 21:02:22,695 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 21:02:22,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 21:02:22,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 21:02:22,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 21:02:22,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 21:02:22,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 21:02:22,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 21:02:22,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 21:02:22,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 21:02:22,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 21:02:22,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 21:02:22,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 21:02:22,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 21:02:22,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 21:02:22,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 21:02:22,711 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 21:02:22,711 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-08 21:02:22,712 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 21:02:22,712 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-08T21:02:22,728 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-08 21:02:22,732 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 21:02:22,732 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T21:02:23,123 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121 2024-12-08T21:02:23,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T21:02:23,197 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f, deleteOnExit=true 2024-12-08T21:02:23,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T21:02:23,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/test.cache.data in system properties and HBase conf 2024-12-08T21:02:23,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T21:02:23,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.log.dir in system properties and HBase conf 2024-12-08T21:02:23,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T21:02:23,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T21:02:23,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T21:02:23,343 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T21:02:23,645 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:02:23,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T21:02:23,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T21:02:23,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T21:02:23,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T21:02:23,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T21:02:23,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T21:02:23,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T21:02:23,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T21:02:23,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T21:02:23,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/nfs.dump.dir in system properties and HBase conf 2024-12-08T21:02:23,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/java.io.tmpdir in system properties and HBase conf 2024-12-08T21:02:23,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T21:02:23,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T21:02:23,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T21:02:25,216 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T21:02:25,299 INFO [Time-limited test {}] log.Log(170): Logging initialized @3670ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T21:02:25,383 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:02:25,442 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T21:02:25,461 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T21:02:25,461 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T21:02:25,463 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T21:02:25,473 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:02:25,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.log.dir/,AVAILABLE} 2024-12-08T21:02:25,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T21:02:25,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/java.io.tmpdir/jetty-localhost-46361-hadoop-hdfs-3_4_1-tests_jar-_-any-9513852657866975560/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T21:02:25,645 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:46361} 2024-12-08T21:02:25,646 INFO [Time-limited test {}] server.Server(415): Started @4017ms 2024-12-08T21:02:26,172 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:02:26,181 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T21:02:26,182 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T21:02:26,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T21:02:26,183 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T21:02:26,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.log.dir/,AVAILABLE} 2024-12-08T21:02:26,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T21:02:26,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/java.io.tmpdir/jetty-localhost-33703-hadoop-hdfs-3_4_1-tests_jar-_-any-14791974915080893222/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T21:02:26,295 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:33703} 2024-12-08T21:02:26,295 INFO [Time-limited test {}] server.Server(415): Started @4667ms 2024-12-08T21:02:26,348 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T21:02:27,225 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/data/data2/current/BP-170596891-172.17.0.2-1733691744660/current, will proceed with Du for space computation calculation, 2024-12-08T21:02:27,225 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/data/data1/current/BP-170596891-172.17.0.2-1733691744660/current, will proceed with Du for space computation calculation, 2024-12-08T21:02:27,259 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T21:02:27,306 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7a116dd7a702e0e with lease ID 0x2cfcf9c68430bb5a: Processing first storage report for DS-758df228-8b6a-4649-b707-10534aa46f90 from datanode DatanodeRegistration(127.0.0.1:40469, datanodeUuid=dd967caa-a9b7-49bc-bc21-50988747a1e3, infoPort=35083, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=1263175019;c=1733691744660) 2024-12-08T21:02:27,308 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7a116dd7a702e0e with lease ID 0x2cfcf9c68430bb5a: from storage DS-758df228-8b6a-4649-b707-10534aa46f90 node DatanodeRegistration(127.0.0.1:40469, datanodeUuid=dd967caa-a9b7-49bc-bc21-50988747a1e3, infoPort=35083, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=1263175019;c=1733691744660), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T21:02:27,308 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7a116dd7a702e0e with lease ID 0x2cfcf9c68430bb5a: Processing first storage report for DS-3881e0a3-7c40-4a05-b76b-4ae52f5384e5 from datanode DatanodeRegistration(127.0.0.1:40469, datanodeUuid=dd967caa-a9b7-49bc-bc21-50988747a1e3, infoPort=35083, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=1263175019;c=1733691744660) 2024-12-08T21:02:27,309 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7a116dd7a702e0e with lease ID 0x2cfcf9c68430bb5a: from storage DS-3881e0a3-7c40-4a05-b76b-4ae52f5384e5 node DatanodeRegistration(127.0.0.1:40469, datanodeUuid=dd967caa-a9b7-49bc-bc21-50988747a1e3, infoPort=35083, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=1263175019;c=1733691744660), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T21:02:27,353 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121 2024-12-08T21:02:27,434 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/zookeeper_0, clientPort=57515, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T21:02:27,445 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57515 2024-12-08T21:02:27,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:27,456 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:27,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741825_1001 (size=7) 2024-12-08T21:02:28,071 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b with version=8 2024-12-08T21:02:28,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/hbase-staging 2024-12-08T21:02:28,163 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T21:02:28,471 INFO [Time-limited test {}] client.ConnectionUtils(128): master/229bab1f9d30:0 server-side Connection retries=9 2024-12-08T21:02:28,482 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:28,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:28,487 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:28,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:28,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:28,638 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T21:02:28,691 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T21:02:28,698 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T21:02:28,701 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:28,721 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 112559 (auto-detected) 2024-12-08T21:02:28,722 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T21:02:28,737 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41621 2024-12-08T21:02:28,756 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41621 connecting to ZooKeeper ensemble=127.0.0.1:57515 2024-12-08T21:02:28,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416210x0, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:28,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41621-0x100074a6af60000 connected 2024-12-08T21:02:28,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:28,928 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:28,935 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:28,939 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b, hbase.cluster.distributed=false 2024-12-08T21:02:28,959 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:28,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41621 2024-12-08T21:02:28,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41621 2024-12-08T21:02:28,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41621 2024-12-08T21:02:28,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41621 2024-12-08T21:02:28,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41621 2024-12-08T21:02:29,051 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/229bab1f9d30:0 server-side Connection retries=9 2024-12-08T21:02:29,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:29,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:29,053 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:02:29,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:02:29,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:02:29,055 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:02:29,057 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:02:29,058 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43999 2024-12-08T21:02:29,059 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43999 connecting to ZooKeeper ensemble=127.0.0.1:57515 2024-12-08T21:02:29,060 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:29,063 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:29,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439990x0, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:02:29,109 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43999-0x100074a6af60001 connected 2024-12-08T21:02:29,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:02:29,115 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:02:29,125 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:02:29,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:02:29,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:02:29,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43999 2024-12-08T21:02:29,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43999 2024-12-08T21:02:29,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43999 2024-12-08T21:02:29,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43999 2024-12-08T21:02:29,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43999 2024-12-08T21:02:29,148 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;229bab1f9d30:41621 2024-12-08T21:02:29,149 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/229bab1f9d30,41621,1733691748281 2024-12-08T21:02:29,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:29,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:29,174 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/229bab1f9d30,41621,1733691748281 2024-12-08T21:02:29,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:29,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:02:29,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:29,200 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T21:02:29,201 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/229bab1f9d30,41621,1733691748281 from backup master directory 2024-12-08T21:02:29,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/229bab1f9d30,41621,1733691748281 2024-12-08T21:02:29,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:29,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:02:29,208 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:29,209 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=229bab1f9d30,41621,1733691748281 2024-12-08T21:02:29,211 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T21:02:29,213 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T21:02:29,265 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/hbase.id] with ID: 552270aa-96a0-412b-95d4-17ea9f9faab1 2024-12-08T21:02:29,266 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/.tmp/hbase.id 2024-12-08T21:02:29,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741826_1002 (size=42) 2024-12-08T21:02:29,680 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/.tmp/hbase.id]:[hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/hbase.id] 2024-12-08T21:02:29,725 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:02:29,729 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T21:02:29,745 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-08T21:02:29,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:29,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:29,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741827_1003 (size=196) 2024-12-08T21:02:30,214 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:02:30,216 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T21:02:30,228 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:02:30,232 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741828_1004 (size=1189) 2024-12-08T21:02:30,683 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store 2024-12-08T21:02:30,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741829_1005 (size=34) 2024-12-08T21:02:31,107 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T21:02:31,109 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:31,111 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:02:31,112 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:31,112 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:31,113 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:02:31,113 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:31,114 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:02:31,115 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691751111Disabling compacts and flushes for region at 1733691751111Disabling writes for close at 1733691751113 (+2 ms)Writing region close event to WAL at 1733691751114 (+1 ms)Closed at 1733691751114 2024-12-08T21:02:31,117 WARN [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/.initializing 2024-12-08T21:02:31,118 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/WALs/229bab1f9d30,41621,1733691748281 2024-12-08T21:02:31,125 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:02:31,140 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=229bab1f9d30%2C41621%2C1733691748281, suffix=, logDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/WALs/229bab1f9d30,41621,1733691748281, archiveDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/oldWALs, maxLogs=10 2024-12-08T21:02:31,169 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/WALs/229bab1f9d30,41621,1733691748281/229bab1f9d30%2C41621%2C1733691748281.1733691751145, exclude list is [], retry=0 2024-12-08T21:02:31,189 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-758df228-8b6a-4649-b707-10534aa46f90,DISK] 2024-12-08T21:02:31,194 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T21:02:31,243 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/WALs/229bab1f9d30,41621,1733691748281/229bab1f9d30%2C41621%2C1733691748281.1733691751145 2024-12-08T21:02:31,245 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083)] 2024-12-08T21:02:31,245 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:02:31,246 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:31,250 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,252 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T21:02:31,387 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:31,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:31,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T21:02:31,402 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:31,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:31,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,418 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T21:02:31,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:31,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:31,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T21:02:31,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:31,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:02:31,429 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,434 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,436 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,444 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,445 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,450 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:31,456 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:02:31,463 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:31,466 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73064929, jitterRate=0.08875228464603424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:31,480 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733691751271Initializing all the Stores at 1733691751278 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691751279 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691751280 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691751280Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691751280Cleaning up temporary data from old regions at 1733691751445 (+165 ms)Region opened successfully at 1733691751480 (+35 ms) 2024-12-08T21:02:31,482 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T21:02:31,544 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11975525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:31,588 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T21:02:31,607 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T21:02:31,607 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T21:02:31,613 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T21:02:31,615 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-08T21:02:31,630 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-08T21:02:31,631 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T21:02:31,671 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T21:02:31,683 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T21:02:31,753 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T21:02:31,756 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T21:02:31,757 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T21:02:31,834 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T21:02:31,836 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T21:02:31,839 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T21:02:32,196 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T21:02:32,200 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T21:02:32,224 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T21:02:32,247 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T21:02:32,274 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T21:02:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:02:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,311 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=229bab1f9d30,41621,1733691748281, sessionid=0x100074a6af60000, setting cluster-up flag (Was=false) 2024-12-08T21:02:32,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,479 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T21:02:32,481 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,41621,1733691748281 2024-12-08T21:02:32,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:02:32,549 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T21:02:32,553 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=229bab1f9d30,41621,1733691748281 2024-12-08T21:02:32,563 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T21:02:32,642 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(746): ClusterId : 552270aa-96a0-412b-95d4-17ea9f9faab1 2024-12-08T21:02:32,646 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:02:32,646 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:32,656 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T21:02:32,662 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T21:02:32,668 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 229bab1f9d30,41621,1733691748281 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T21:02:32,675 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:02:32,676 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/229bab1f9d30:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/229bab1f9d30:0, corePoolSize=10, maxPoolSize=10 2024-12-08T21:02:32,676 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:32,677 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,678 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733691782678 2024-12-08T21:02:32,680 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T21:02:32,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T21:02:32,684 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:32,684 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T21:02:32,685 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T21:02:32,687 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T21:02:32,687 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T21:02:32,688 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T21:02:32,689 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:02:32,690 DEBUG [RS:0;229bab1f9d30:43999 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5964f91d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=229bab1f9d30/172.17.0.2:0 2024-12-08T21:02:32,689 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,696 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:32,696 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T21:02:32,697 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T21:02:32,698 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T21:02:32,698 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T21:02:32,701 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T21:02:32,701 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T21:02:32,708 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691752703,5,FailOnTimeoutGroup] 2024-12-08T21:02:32,709 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691752708,5,FailOnTimeoutGroup] 2024-12-08T21:02:32,709 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,709 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T21:02:32,710 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;229bab1f9d30:43999 2024-12-08T21:02:32,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741831_1007 (size=1321) 2024-12-08T21:02:32,711 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,711 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,713 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T21:02:32,713 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:02:32,715 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:02:32,715 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:02:32,715 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:02:32,718 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(2659): reportForDuty to master=229bab1f9d30,41621,1733691748281 with port=43999, startcode=1733691749022 2024-12-08T21:02:32,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741832_1008 (size=32) 2024-12-08T21:02:32,732 DEBUG [RS:0;229bab1f9d30:43999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:02:32,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44971, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:02:32,774 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41621 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41621 {}] master.ServerManager(517): Registering regionserver=229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,792 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:02:32,792 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42587 2024-12-08T21:02:32,792 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:02:32,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:02:32,813 DEBUG [RS:0;229bab1f9d30:43999 {}] zookeeper.ZKUtil(111): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,813 WARN [RS:0;229bab1f9d30:43999 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:02:32,813 INFO [RS:0;229bab1f9d30:43999 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:02:32,814 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,816 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [229bab1f9d30,43999,1733691749022] 2024-12-08T21:02:32,839 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:02:32,851 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:02:32,856 INFO [RS:0;229bab1f9d30:43999 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:02:32,856 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,857 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:02:32,862 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:02:32,863 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,864 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/229bab1f9d30:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,865 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/229bab1f9d30:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:02:32,866 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:32,866 DEBUG [RS:0;229bab1f9d30:43999 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/229bab1f9d30:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:02:32,867 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,867 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,867 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,867 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,868 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,868 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,43999,1733691749022-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:02:32,888 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:02:32,889 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,43999,1733691749022-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,890 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,890 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.Replication(171): 229bab1f9d30,43999,1733691749022 started 2024-12-08T21:02:32,908 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:02:32,908 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1482): Serving as 229bab1f9d30,43999,1733691749022, RpcServer on 229bab1f9d30/172.17.0.2:43999, sessionid=0x100074a6af60001 2024-12-08T21:02:32,909 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:02:32,909 DEBUG [RS:0;229bab1f9d30:43999 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,909 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,43999,1733691749022' 2024-12-08T21:02:32,909 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:02:32,910 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:02:32,911 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:02:32,911 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:02:32,911 DEBUG [RS:0;229bab1f9d30:43999 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 229bab1f9d30,43999,1733691749022 2024-12-08T21:02:32,911 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '229bab1f9d30,43999,1733691749022' 2024-12-08T21:02:32,911 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:02:32,912 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:02:32,912 DEBUG [RS:0;229bab1f9d30:43999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:02:32,913 INFO [RS:0;229bab1f9d30:43999 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:02:32,913 INFO [RS:0;229bab1f9d30:43999 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:02:33,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:02:33,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T21:02:33,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:02:33,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:33,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:33,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T21:02:33,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:02:33,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:33,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:33,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T21:02:33,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:02:33,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:33,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:33,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T21:02:33,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:02:33,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:02:33,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:02:33,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:02:33,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740 2024-12-08T21:02:33,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740 2024-12-08T21:02:33,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:02:33,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:02:33,156 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:02:33,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:02:33,163 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:02:33,164 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66507117, jitterRate=-0.008966729044914246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:02:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733691753128Initializing all the Stores at 1733691753131 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691753132 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691753132Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691753132Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691753132Cleaning up temporary data from old regions at 1733691753155 (+23 ms)Region opened successfully at 1733691753166 (+11 ms) 2024-12-08T21:02:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:02:33,167 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:02:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:02:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:02:33,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:02:33,168 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:02:33,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691753167Disabling compacts and flushes for region at 1733691753167Disabling writes for close at 1733691753167Writing region close event to WAL at 1733691753168 (+1 ms)Closed at 1733691753168 2024-12-08T21:02:33,172 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:02:33,172 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T21:02:33,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T21:02:33,186 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:02:33,190 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T21:02:33,341 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:34,342 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:35,343 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:36,344 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:37,345 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:38,346 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:38,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T21:02:38,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T21:02:39,347 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:40,347 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:41,348 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:42,349 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:43,350 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:44,351 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:45,352 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:46,353 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:47,354 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:48,355 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:49,355 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:50,356 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:51,357 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:52,358 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:53,358 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:54,359 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:55,360 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:56,361 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:57,350 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:02:57,362 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:58,363 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:02:59,364 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:03:00,365 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:03:01,366 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:03:02,367 WARN [229bab1f9d30:41621 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T21:03:02,927 INFO [RS:0;229bab1f9d30:43999 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:03:02,931 INFO [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=229bab1f9d30%2C43999%2C1733691749022, suffix=, logDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022, archiveDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/oldWALs, maxLogs=32 2024-12-08T21:03:02,945 DEBUG [RS:0;229bab1f9d30:43999 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022/229bab1f9d30%2C43999%2C1733691749022.1733691782933, exclude list is [], retry=0 2024-12-08T21:03:02,949 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-758df228-8b6a-4649-b707-10534aa46f90,DISK] 2024-12-08T21:03:02,953 INFO [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022/229bab1f9d30%2C43999%2C1733691749022.1733691782933 2024-12-08T21:03:02,953 DEBUG [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083)] 2024-12-08T21:03:03,121 DEBUG [229bab1f9d30:41621 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T21:03:03,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:03,143 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,43999,1733691749022, state=OPENING 2024-12-08T21:03:03,180 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T21:03:03,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:03:03,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:03:03,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:03:03,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:03:03,194 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:03:03,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:03,377 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:03:03,380 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52635, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:03:03,389 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T21:03:03,390 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:03:03,390 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T21:03:03,393 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=229bab1f9d30%2C43999%2C1733691749022.meta, suffix=.meta, logDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022, archiveDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/oldWALs, maxLogs=32 2024-12-08T21:03:03,409 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022/229bab1f9d30%2C43999%2C1733691749022.meta.1733691783395.meta, exclude list is [], retry=0 2024-12-08T21:03:03,412 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-758df228-8b6a-4649-b707-10534aa46f90,DISK] 2024-12-08T21:03:03,416 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022/229bab1f9d30%2C43999%2C1733691749022.meta.1733691783395.meta 2024-12-08T21:03:03,416 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083)] 2024-12-08T21:03:03,416 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:03,418 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T21:03:03,421 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T21:03:03,426 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T21:03:03,431 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T21:03:03,431 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:03,432 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T21:03:03,432 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T21:03:03,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T21:03:03,436 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:03:03,436 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:03,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:03:03,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T21:03:03,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:03:03,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:03,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:03:03,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T21:03:03,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:03:03,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:03,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:03:03,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T21:03:03,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:03:03,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:03,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:03:03,445 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:03:03,447 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740 2024-12-08T21:03:03,450 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740 2024-12-08T21:03:03,453 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:03:03,453 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:03:03,453 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:03:03,456 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:03:03,458 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71969096, jitterRate=0.07242310047149658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:03:03,458 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T21:03:03,459 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733691783432Writing region info on filesystem at 1733691783432Initializing all the Stores at 1733691783434 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691783434Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691783435 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691783435Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733691783435Cleaning up temporary data from old regions at 1733691783453 (+18 ms)Running coprocessor post-open hooks at 1733691783458 (+5 ms)Region opened successfully at 1733691783459 (+1 ms) 2024-12-08T21:03:03,465 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733691783370 2024-12-08T21:03:03,476 DEBUG [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T21:03:03,476 INFO [RS_OPEN_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T21:03:03,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:03,480 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 229bab1f9d30,43999,1733691749022, state=OPEN 2024-12-08T21:03:03,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:03:03,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:03:03,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:03:03,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:03:03,582 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:03,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T21:03:03,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=229bab1f9d30,43999,1733691749022 in 386 msec 2024-12-08T21:03:03,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T21:03:03,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.4160 sec 2024-12-08T21:03:03,600 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:03:03,600 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T21:03:03,616 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:03:03,617 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,43999,1733691749022, seqNum=-1] 2024-12-08T21:03:03,634 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:03:03,636 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:03:03,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0600 sec 2024-12-08T21:03:03,655 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733691783655, completionTime=-1 2024-12-08T21:03:03,659 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T21:03:03,659 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T21:03:03,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T21:03:03,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733691843681 2024-12-08T21:03:03,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733691903681 2024-12-08T21:03:03,681 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-08T21:03:03,683 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,683 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,683 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,684 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-229bab1f9d30:41621, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,685 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,685 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,690 DEBUG [master/229bab1f9d30:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T21:03:03,709 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 34.500sec 2024-12-08T21:03:03,710 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T21:03:03,711 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T21:03:03,712 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T21:03:03,713 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T21:03:03,713 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T21:03:03,713 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:03:03,714 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T21:03:03,721 DEBUG [master/229bab1f9d30:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T21:03:03,722 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T21:03:03,722 INFO [master/229bab1f9d30:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=229bab1f9d30,41621,1733691748281-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:03:03,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2560d350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,798 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T21:03:03,799 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T21:03:03,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 229bab1f9d30,41621,-1 for getting cluster id 2024-12-08T21:03:03,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:03:03,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '552270aa-96a0-412b-95d4-17ea9f9faab1' 2024-12-08T21:03:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:03:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "552270aa-96a0-412b-95d4-17ea9f9faab1" 2024-12-08T21:03:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794ea6fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [229bab1f9d30,41621,-1] 2024-12-08T21:03:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:03:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:03:03,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:03:03,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456a05a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,821 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:03:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,43999,1733691749022, seqNum=-1] 2024-12-08T21:03:03,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:03:03,830 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:03:03,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=229bab1f9d30,41621,1733691748281 2024-12-08T21:03:03,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a73b085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 229bab1f9d30,41621,-1 for getting cluster id 2024-12-08T21:03:03,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:03:03,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '552270aa-96a0-412b-95d4-17ea9f9faab1' 2024-12-08T21:03:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:03:03,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "552270aa-96a0-412b-95d4-17ea9f9faab1" 2024-12-08T21:03:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1601c7a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [229bab1f9d30,41621,-1] 2024-12-08T21:03:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:03:03,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:03:03,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:03:03,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1939ea58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:03:03,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=17097 2024-12-08T21:03:03,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:03:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 229bab1f9d30,41621,1733691748281 2024-12-08T21:03:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cb05583 2024-12-08T21:03:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:03:03,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:03:03,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-08T21:03:03,922 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:03,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-08T21:03:03,924 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:03,930 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:03:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741835_1011 (size=428) 2024-12-08T21:03:04,355 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 879a1685c7f2dd2f7b50136f3562a731, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:04,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741836_1012 (size=79) 2024-12-08T21:03:04,367 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:04,367 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 879a1685c7f2dd2f7b50136f3562a731, disabling compactions & flushes 2024-12-08T21:03:04,367 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,367 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,367 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. after waiting 0 ms 2024-12-08T21:03:04,368 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,368 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,368 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 879a1685c7f2dd2f7b50136f3562a731: Waiting for close lock at 1733691784367Disabling compacts and flushes for region at 1733691784367Disabling writes for close at 1733691784368 (+1 ms)Writing region close event to WAL at 1733691784368Closed at 1733691784368 2024-12-08T21:03:04,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:04,376 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733691784370"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691784370"}]},"ts":"1733691784370"} 2024-12-08T21:03:04,381 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:04,382 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:04,385 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691784383"}]},"ts":"1733691784383"} 2024-12-08T21:03:04,389 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-08T21:03:04,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=879a1685c7f2dd2f7b50136f3562a731, ASSIGN}] 2024-12-08T21:03:04,394 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=879a1685c7f2dd2f7b50136f3562a731, ASSIGN 2024-12-08T21:03:04,396 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=879a1685c7f2dd2f7b50136f3562a731, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:04,549 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=879a1685c7f2dd2f7b50136f3562a731, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:04,560 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=879a1685c7f2dd2f7b50136f3562a731, ASSIGN because future has completed 2024-12-08T21:03:04,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 879a1685c7f2dd2f7b50136f3562a731, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:04,733 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,733 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 879a1685c7f2dd2f7b50136f3562a731, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:04,734 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,734 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:04,734 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,734 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,737 INFO [StoreOpener-879a1685c7f2dd2f7b50136f3562a731-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,739 INFO [StoreOpener-879a1685c7f2dd2f7b50136f3562a731-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 879a1685c7f2dd2f7b50136f3562a731 columnFamilyName testFamily 2024-12-08T21:03:04,740 DEBUG [StoreOpener-879a1685c7f2dd2f7b50136f3562a731-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:04,741 INFO [StoreOpener-879a1685c7f2dd2f7b50136f3562a731-1 {}] regionserver.HStore(327): Store=879a1685c7f2dd2f7b50136f3562a731/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:04,741 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,742 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,743 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,744 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,744 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,746 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,749 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:04,750 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 879a1685c7f2dd2f7b50136f3562a731; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65647342, jitterRate=-0.021778374910354614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:04,750 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:03:04,751 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 879a1685c7f2dd2f7b50136f3562a731: Running coprocessor pre-open hook at 1733691784735Writing region info on filesystem at 1733691784735Initializing all the Stores at 1733691784736 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691784736Cleaning up temporary data from old regions at 1733691784744 (+8 ms)Running coprocessor post-open hooks at 1733691784750 (+6 ms)Region opened successfully at 1733691784751 (+1 ms) 2024-12-08T21:03:04,753 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731., pid=6, masterSystemTime=1733691784718 2024-12-08T21:03:04,756 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,756 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:03:04,757 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=879a1685c7f2dd2f7b50136f3562a731, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:04,762 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 879a1685c7f2dd2f7b50136f3562a731, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:04,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T21:03:04,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 879a1685c7f2dd2f7b50136f3562a731, server=229bab1f9d30,43999,1733691749022 in 202 msec 2024-12-08T21:03:04,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T21:03:04,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=879a1685c7f2dd2f7b50136f3562a731, ASSIGN in 376 msec 2024-12-08T21:03:04,772 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:04,773 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691784773"}]},"ts":"1733691784773"} 2024-12-08T21:03:04,776 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-08T21:03:04,777 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:04,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 864 msec 2024-12-08T21:03:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:03:04,960 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-08T21:03:04,960 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-08T21:03:04,964 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:04,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-08T21:03:04,971 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:04,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-08T21:03:04,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:03:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=229bab1f9d30,43999,1733691749022, seqNum=-1] 2024-12-08T21:03:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:03:04,978 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:03:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:04,998 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:05,003 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-08T21:03:05,009 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-08T21:03:05,014 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-08T21:03:05,016 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-08T21:03:05,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=4749821091608789000 2024-12-08T21:03:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691788015, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:05,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-08T21:03:05,041 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-08T21:03:05,042 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:05,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:05,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=4749821091608789002 2024-12-08T21:03:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691788043, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:05,048 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:05,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=4749821091608789003 2024-12-08T21:03:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691788047, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:05,052 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:05,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=4749821091608789004 2024-12-08T21:03:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691788051, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:05,056 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:05,065 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_993132359_22 at /127.0.0.1:59702 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=293 (was 293), ProcessCount=11 (was 11), AvailableMemoryMB=17052 (was 17097) 2024-12-08T21:03:05,071 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=293, ProcessCount=11, AvailableMemoryMB=17052 2024-12-08T21:03:05,075 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:05,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-08T21:03:05,079 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:05,080 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:05,080 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-08T21:03:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:03:05,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:05,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741837_1013 (size=414) 2024-12-08T21:03:05,499 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d301d980e330faad26a3ed42cc20b3c5, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:05,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741838_1014 (size=65) 2024-12-08T21:03:05,911 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:05,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d301d980e330faad26a3ed42cc20b3c5, disabling compactions & flushes 2024-12-08T21:03:05,912 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:05,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:05,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. after waiting 0 ms 2024-12-08T21:03:05,912 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:05,912 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:05,913 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d301d980e330faad26a3ed42cc20b3c5: Waiting for close lock at 1733691785912Disabling compacts and flushes for region at 1733691785912Disabling writes for close at 1733691785912Writing region close event to WAL at 1733691785912Closed at 1733691785912 2024-12-08T21:03:05,917 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:05,918 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733691785917"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691785917"}]},"ts":"1733691785917"} 2024-12-08T21:03:05,922 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:05,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:05,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691785924"}]},"ts":"1733691785924"} 2024-12-08T21:03:05,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-08T21:03:05,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d301d980e330faad26a3ed42cc20b3c5, ASSIGN}] 2024-12-08T21:03:05,930 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d301d980e330faad26a3ed42cc20b3c5, ASSIGN 2024-12-08T21:03:05,932 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d301d980e330faad26a3ed42cc20b3c5, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:06,083 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d301d980e330faad26a3ed42cc20b3c5, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:06,087 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d301d980e330faad26a3ed42cc20b3c5, ASSIGN because future has completed 2024-12-08T21:03:06,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d301d980e330faad26a3ed42cc20b3c5, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:03:06,257 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:06,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d301d980e330faad26a3ed42cc20b3c5, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:06,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:06,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,258 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,260 INFO [StoreOpener-d301d980e330faad26a3ed42cc20b3c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,262 INFO [StoreOpener-d301d980e330faad26a3ed42cc20b3c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d301d980e330faad26a3ed42cc20b3c5 columnFamilyName testFamily 2024-12-08T21:03:06,262 DEBUG [StoreOpener-d301d980e330faad26a3ed42cc20b3c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:06,262 INFO [StoreOpener-d301d980e330faad26a3ed42cc20b3c5-1 {}] regionserver.HStore(327): Store=d301d980e330faad26a3ed42cc20b3c5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:06,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,263 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:06,271 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened d301d980e330faad26a3ed42cc20b3c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60743966, jitterRate=-0.09484437108039856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:06,271 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:03:06,272 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d301d980e330faad26a3ed42cc20b3c5: Running coprocessor pre-open hook at 1733691786258Writing region info on filesystem at 1733691786258Initializing all the Stores at 1733691786259 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691786260 (+1 ms)Cleaning up temporary data from old regions at 1733691786265 (+5 ms)Running coprocessor post-open hooks at 1733691786271 (+6 ms)Region opened successfully at 1733691786272 (+1 ms) 2024-12-08T21:03:06,273 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., pid=9, masterSystemTime=1733691786243 2024-12-08T21:03:06,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:06,276 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:03:06,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d301d980e330faad26a3ed42cc20b3c5, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:06,280 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d301d980e330faad26a3ed42cc20b3c5, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:06,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-08T21:03:06,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure d301d980e330faad26a3ed42cc20b3c5, server=229bab1f9d30,43999,1733691749022 in 194 msec 2024-12-08T21:03:06,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T21:03:06,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d301d980e330faad26a3ed42cc20b3c5, ASSIGN in 357 msec 2024-12-08T21:03:06,291 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:06,291 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691786291"}]},"ts":"1733691786291"} 2024-12-08T21:03:06,294 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-08T21:03:06,295 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:06,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2200 sec 2024-12-08T21:03:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:03:08,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-08T21:03:08,117 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-08T21:03:08,118 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:08,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-08T21:03:08,127 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:08,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-08T21:03:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:08,144 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:08,144 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-08T21:03:08,147 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T21:03:08,149 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-08T21:03:08,970 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:08,992 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T21:03:08,992 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:08,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:03:08,993 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T21:03:08,994 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-08T21:03:08,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:08,994 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-08T21:03:08,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:08,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-08T21:03:08,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T21:03:08,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-08T21:03:14,168 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-12-08T21:03:14,169 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-12-08T21:03:14,169 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:03:14,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 293), ProcessCount=11 (was 11), AvailableMemoryMB=17039 (was 17052) 2024-12-08T21:03:14,185 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=17039 2024-12-08T21:03:14,188 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:14,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-08T21:03:14,191 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:14,191 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:14,192 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-08T21:03:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T21:03:14,193 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:14,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741839_1015 (size=412) 2024-12-08T21:03:14,608 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38588e6d1cec3f8d460fda2df0ad9f8d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741840_1016 (size=63) 2024-12-08T21:03:14,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-08T21:03:15,050 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:15,051 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 38588e6d1cec3f8d460fda2df0ad9f8d, disabling compactions & flushes 2024-12-08T21:03:15,051 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,051 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,051 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. after waiting 0 ms 2024-12-08T21:03:15,051 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,051 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,051 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38588e6d1cec3f8d460fda2df0ad9f8d: Waiting for close lock at 1733691795050Disabling compacts and flushes for region at 1733691795050Disabling writes for close at 1733691795051 (+1 ms)Writing region close event to WAL at 1733691795051Closed at 1733691795051 2024-12-08T21:03:15,054 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:15,055 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733691795054"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691795054"}]},"ts":"1733691795054"} 2024-12-08T21:03:15,060 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:15,062 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:15,062 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691795062"}]},"ts":"1733691795062"} 2024-12-08T21:03:15,065 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-08T21:03:15,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38588e6d1cec3f8d460fda2df0ad9f8d, ASSIGN}] 2024-12-08T21:03:15,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38588e6d1cec3f8d460fda2df0ad9f8d, ASSIGN 2024-12-08T21:03:15,068 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38588e6d1cec3f8d460fda2df0ad9f8d, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T21:03:15,219 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=38588e6d1cec3f8d460fda2df0ad9f8d, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:15,226 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38588e6d1cec3f8d460fda2df0ad9f8d, ASSIGN because future has completed 2024-12-08T21:03:15,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38588e6d1cec3f8d460fda2df0ad9f8d, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:15,395 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 38588e6d1cec3f8d460fda2df0ad9f8d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:15,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:15,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,396 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,399 INFO [StoreOpener-38588e6d1cec3f8d460fda2df0ad9f8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,401 INFO [StoreOpener-38588e6d1cec3f8d460fda2df0ad9f8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38588e6d1cec3f8d460fda2df0ad9f8d columnFamilyName testFamily 2024-12-08T21:03:15,401 DEBUG [StoreOpener-38588e6d1cec3f8d460fda2df0ad9f8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:15,402 INFO [StoreOpener-38588e6d1cec3f8d460fda2df0ad9f8d-1 {}] regionserver.HStore(327): Store=38588e6d1cec3f8d460fda2df0ad9f8d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:15,403 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,404 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,406 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,409 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,413 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:15,414 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 38588e6d1cec3f8d460fda2df0ad9f8d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58972012, jitterRate=-0.12124854326248169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:15,414 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:03:15,415 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 38588e6d1cec3f8d460fda2df0ad9f8d: Running coprocessor pre-open hook at 1733691795396Writing region info on filesystem at 1733691795396Initializing all the Stores at 1733691795398 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691795398Cleaning up temporary data from old regions at 1733691795406 (+8 ms)Running coprocessor post-open hooks at 1733691795414 (+8 ms)Region opened successfully at 1733691795415 (+1 ms) 2024-12-08T21:03:15,417 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d., pid=12, masterSystemTime=1733691795382 2024-12-08T21:03:15,420 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,420 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:03:15,421 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=38588e6d1cec3f8d460fda2df0ad9f8d, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:15,424 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38588e6d1cec3f8d460fda2df0ad9f8d, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:15,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T21:03:15,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 38588e6d1cec3f8d460fda2df0ad9f8d, server=229bab1f9d30,43999,1733691749022 in 198 msec 2024-12-08T21:03:15,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-08T21:03:15,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=38588e6d1cec3f8d460fda2df0ad9f8d, ASSIGN in 364 msec 2024-12-08T21:03:15,434 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:15,434 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691795434"}]},"ts":"1733691795434"} 2024-12-08T21:03:15,437 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-08T21:03:15,439 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:15,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2510 sec 2024-12-08T21:03:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-08T21:03:17,238 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-08T21:03:17,238 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-08T21:03:17,239 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:17,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-08T21:03:17,249 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:17,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-08T21:03:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:17,263 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:17,263 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-08T21:03:17,266 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T21:03:17,268 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T21:03:17,270 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T21:03:17,270 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-08T21:03:18,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-08T21:03:18,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:20,919 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:21,000 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-08T21:03:23,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789013' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T21:03:23,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57664 deadline: 1733691809283, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789013' does not exist 2024-12-08T21:03:23,290 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 229bab1f9d30,43999,1733691749022 for closing scanner id = 4749821091608789013 for 38588e6d1cec3f8d460fda2df0ad9f8d of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789013' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:03:23,293 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=17007 (was 17039) 2024-12-08T21:03:23,301 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=209, ProcessCount=11, AvailableMemoryMB=17006 2024-12-08T21:03:23,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-08T21:03:23,307 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:23,307 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:23,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-08T21:03:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:03:23,308 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:23,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741841_1017 (size=423) 2024-12-08T21:03:23,318 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a507a1df86c7c06aed001d2290ba682, NAME => 'testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:23,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741842_1018 (size=74) 2024-12-08T21:03:23,728 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:23,729 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 1a507a1df86c7c06aed001d2290ba682, disabling compactions & flushes 2024-12-08T21:03:23,729 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:23,729 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:23,729 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. after waiting 0 ms 2024-12-08T21:03:23,729 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:23,729 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:23,730 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a507a1df86c7c06aed001d2290ba682: Waiting for close lock at 1733691803729Disabling compacts and flushes for region at 1733691803729Disabling writes for close at 1733691803729Writing region close event to WAL at 1733691803729Closed at 1733691803729 2024-12-08T21:03:23,733 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:23,734 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733691803733"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691803733"}]},"ts":"1733691803733"} 2024-12-08T21:03:23,739 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:23,741 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:23,742 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691803741"}]},"ts":"1733691803741"} 2024-12-08T21:03:23,745 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-08T21:03:23,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1a507a1df86c7c06aed001d2290ba682, ASSIGN}] 2024-12-08T21:03:23,748 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1a507a1df86c7c06aed001d2290ba682, ASSIGN 2024-12-08T21:03:23,749 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1a507a1df86c7c06aed001d2290ba682, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-08T21:03:23,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1a507a1df86c7c06aed001d2290ba682, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:23,911 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1a507a1df86c7c06aed001d2290ba682, ASSIGN because future has completed 2024-12-08T21:03:23,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a507a1df86c7c06aed001d2290ba682, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:24,076 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:24,077 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a507a1df86c7c06aed001d2290ba682, NAME => 'testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:24,078 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,078 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:24,078 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,078 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,081 INFO [StoreOpener-1a507a1df86c7c06aed001d2290ba682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,084 INFO [StoreOpener-1a507a1df86c7c06aed001d2290ba682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a507a1df86c7c06aed001d2290ba682 columnFamilyName testFamily 2024-12-08T21:03:24,084 DEBUG [StoreOpener-1a507a1df86c7c06aed001d2290ba682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:24,085 INFO [StoreOpener-1a507a1df86c7c06aed001d2290ba682-1 {}] regionserver.HStore(327): Store=1a507a1df86c7c06aed001d2290ba682/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:24,085 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,086 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,087 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,087 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,087 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,090 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,093 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:24,093 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 1a507a1df86c7c06aed001d2290ba682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73048785, jitterRate=0.08851172029972076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:24,094 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:03:24,094 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 1a507a1df86c7c06aed001d2290ba682: Running coprocessor pre-open hook at 1733691804079Writing region info on filesystem at 1733691804079Initializing all the Stores at 1733691804080 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691804080Cleaning up temporary data from old regions at 1733691804087 (+7 ms)Running coprocessor post-open hooks at 1733691804094 (+7 ms)Region opened successfully at 1733691804094 2024-12-08T21:03:24,096 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682., pid=15, masterSystemTime=1733691804064 2024-12-08T21:03:24,098 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:24,098 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:03:24,099 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1a507a1df86c7c06aed001d2290ba682, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:24,102 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a507a1df86c7c06aed001d2290ba682, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:24,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-08T21:03:24,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a507a1df86c7c06aed001d2290ba682, server=229bab1f9d30,43999,1733691749022 in 193 msec 2024-12-08T21:03:24,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T21:03:24,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=1a507a1df86c7c06aed001d2290ba682, ASSIGN in 362 msec 2024-12-08T21:03:24,111 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:24,111 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691804111"}]},"ts":"1733691804111"} 2024-12-08T21:03:24,113 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-08T21:03:24,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:24,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 811 msec 2024-12-08T21:03:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:03:24,327 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-08T21:03:24,327 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-08T21:03:24,328 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:24,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-08T21:03:24,337 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:24,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-08T21:03:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:24,350 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:24,350 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-08T21:03:24,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-08T21:03:24,356 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-08T21:03:24,358 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-08T21:03:24,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=4749821091608789017 2024-12-08T21:03:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691807357, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:24,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-08T21:03:24,364 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-08T21:03:24,364 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:24,366 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:24,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=4749821091608789019 2024-12-08T21:03:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691807366, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:24,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:24,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=4749821091608789020 2024-12-08T21:03:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691807371, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:24,376 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:24,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=4749821091608789021 2024-12-08T21:03:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:57664 deadline: 1733691807375, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-08T21:03:24,380 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-08T21:03:24,387 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_993132359_22 at /127.0.0.1:52694 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 209), ProcessCount=11 (was 11), AvailableMemoryMB=17003 (was 17006) 2024-12-08T21:03:24,396 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=209, ProcessCount=11, AvailableMemoryMB=17003 2024-12-08T21:03:24,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-08T21:03:24,403 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:24,403 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:24,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-08T21:03:24,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-08T21:03:24,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741843_1019 (size=418) 2024-12-08T21:03:24,820 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 319e5659b18dc5e2899c9e019b76dea1, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741844_1020 (size=69) 2024-12-08T21:03:25,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:25,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 319e5659b18dc5e2899c9e019b76dea1, disabling compactions & flushes 2024-12-08T21:03:25,234 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. after waiting 0 ms 2024-12-08T21:03:25,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,234 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,234 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 319e5659b18dc5e2899c9e019b76dea1: Waiting for close lock at 1733691805234Disabling compacts and flushes for region at 1733691805234Disabling writes for close at 1733691805234Writing region close event to WAL at 1733691805234Closed at 1733691805234 2024-12-08T21:03:25,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:25,239 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733691805238"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691805238"}]},"ts":"1733691805238"} 2024-12-08T21:03:25,243 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:25,245 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:25,245 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691805245"}]},"ts":"1733691805245"} 2024-12-08T21:03:25,249 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-08T21:03:25,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=319e5659b18dc5e2899c9e019b76dea1, ASSIGN}] 2024-12-08T21:03:25,251 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=319e5659b18dc5e2899c9e019b76dea1, ASSIGN 2024-12-08T21:03:25,253 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=319e5659b18dc5e2899c9e019b76dea1, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:25,404 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=319e5659b18dc5e2899c9e019b76dea1, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:25,411 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=319e5659b18dc5e2899c9e019b76dea1, ASSIGN because future has completed 2024-12-08T21:03:25,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 319e5659b18dc5e2899c9e019b76dea1, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-08T21:03:25,575 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,575 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 319e5659b18dc5e2899c9e019b76dea1, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:25,575 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,576 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:25,576 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,576 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,578 INFO [StoreOpener-319e5659b18dc5e2899c9e019b76dea1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,580 INFO [StoreOpener-319e5659b18dc5e2899c9e019b76dea1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 319e5659b18dc5e2899c9e019b76dea1 columnFamilyName testFamily 2024-12-08T21:03:25,580 DEBUG [StoreOpener-319e5659b18dc5e2899c9e019b76dea1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:25,581 INFO [StoreOpener-319e5659b18dc5e2899c9e019b76dea1-1 {}] regionserver.HStore(327): Store=319e5659b18dc5e2899c9e019b76dea1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:25,581 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,582 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,583 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,584 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,584 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,586 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,590 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:25,590 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 319e5659b18dc5e2899c9e019b76dea1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67697025, jitterRate=0.008764281868934631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:25,590 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:03:25,591 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 319e5659b18dc5e2899c9e019b76dea1: Running coprocessor pre-open hook at 1733691805576Writing region info on filesystem at 1733691805576Initializing all the Stores at 1733691805577 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691805577Cleaning up temporary data from old regions at 1733691805584 (+7 ms)Running coprocessor post-open hooks at 1733691805590 (+6 ms)Region opened successfully at 1733691805591 (+1 ms) 2024-12-08T21:03:25,592 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., pid=18, masterSystemTime=1733691805566 2024-12-08T21:03:25,595 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,595 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:03:25,596 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=319e5659b18dc5e2899c9e019b76dea1, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:25,599 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 319e5659b18dc5e2899c9e019b76dea1, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:25,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T21:03:25,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 319e5659b18dc5e2899c9e019b76dea1, server=229bab1f9d30,43999,1733691749022 in 189 msec 2024-12-08T21:03:25,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-08T21:03:25,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=319e5659b18dc5e2899c9e019b76dea1, ASSIGN in 354 msec 2024-12-08T21:03:25,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:25,608 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691805608"}]},"ts":"1733691805608"} 2024-12-08T21:03:25,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-08T21:03:25,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:25,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2120 sec 2024-12-08T21:03:27,003 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-08T21:03:27,005 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-08T21:03:27,350 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:03:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-08T21:03:27,447 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-08T21:03:27,447 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-08T21:03:27,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:27,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-08T21:03:27,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:27,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-08T21:03:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:27,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:27,477 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-08T21:03:27,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T21:03:27,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T21:03:28,485 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-12-08T21:03:28,485 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1005ms, rpcTimeout=1000ms 2024-12-08T21:03:28,485 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:03:28,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-08T21:03:28,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:28,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-08T21:03:28,690 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-08T21:03:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:29,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T21:03:29,582 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:30,486 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=982ms 2024-12-08T21:03:30,486 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=982ms 2024-12-08T21:03:30,486 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:03:30,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=193 (was 209), ProcessCount=11 (was 11), AvailableMemoryMB=17002 (was 17003) 2024-12-08T21:03:30,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=193, ProcessCount=11, AvailableMemoryMB=17002 2024-12-08T21:03:30,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-08T21:03:30,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:30,518 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:30,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-08T21:03:30,520 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:03:30,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741845_1021 (size=416) 2024-12-08T21:03:30,531 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4788c4822a4ce08ce43f13bc3e5d2847, NAME => 'testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741846_1022 (size=67) 2024-12-08T21:03:30,943 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:30,943 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4788c4822a4ce08ce43f13bc3e5d2847, disabling compactions & flushes 2024-12-08T21:03:30,943 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:30,943 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:30,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. after waiting 0 ms 2024-12-08T21:03:30,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:30,944 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:30,945 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4788c4822a4ce08ce43f13bc3e5d2847: Waiting for close lock at 1733691810943Disabling compacts and flushes for region at 1733691810943Disabling writes for close at 1733691810944 (+1 ms)Writing region close event to WAL at 1733691810944Closed at 1733691810944 2024-12-08T21:03:30,950 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:30,950 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733691810950"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691810950"}]},"ts":"1733691810950"} 2024-12-08T21:03:30,953 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:30,955 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:30,955 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691810955"}]},"ts":"1733691810955"} 2024-12-08T21:03:30,957 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-08T21:03:30,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=4788c4822a4ce08ce43f13bc3e5d2847, ASSIGN}] 2024-12-08T21:03:30,959 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=4788c4822a4ce08ce43f13bc3e5d2847, ASSIGN 2024-12-08T21:03:30,961 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=4788c4822a4ce08ce43f13bc3e5d2847, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:31,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-08T21:03:31,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=4788c4822a4ce08ce43f13bc3e5d2847, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:31,120 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=4788c4822a4ce08ce43f13bc3e5d2847, ASSIGN because future has completed 2024-12-08T21:03:31,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4788c4822a4ce08ce43f13bc3e5d2847, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:31,285 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:31,286 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 4788c4822a4ce08ce43f13bc3e5d2847, NAME => 'testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:31,286 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,287 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:31,287 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,287 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,289 INFO [StoreOpener-4788c4822a4ce08ce43f13bc3e5d2847-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,291 INFO [StoreOpener-4788c4822a4ce08ce43f13bc3e5d2847-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4788c4822a4ce08ce43f13bc3e5d2847 columnFamilyName testFamily 2024-12-08T21:03:31,291 DEBUG [StoreOpener-4788c4822a4ce08ce43f13bc3e5d2847-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:31,292 INFO [StoreOpener-4788c4822a4ce08ce43f13bc3e5d2847-1 {}] regionserver.HStore(327): Store=4788c4822a4ce08ce43f13bc3e5d2847/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:31,292 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,293 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,293 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,294 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,294 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,296 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,299 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:31,300 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 4788c4822a4ce08ce43f13bc3e5d2847; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74102459, jitterRate=0.104212686419487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:31,300 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:03:31,301 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 4788c4822a4ce08ce43f13bc3e5d2847: Running coprocessor pre-open hook at 1733691811287Writing region info on filesystem at 1733691811287Initializing all the Stores at 1733691811289 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691811289Cleaning up temporary data from old regions at 1733691811294 (+5 ms)Running coprocessor post-open hooks at 1733691811300 (+6 ms)Region opened successfully at 1733691811301 (+1 ms) 2024-12-08T21:03:31,303 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., pid=21, masterSystemTime=1733691811274 2024-12-08T21:03:31,305 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:31,305 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:03:31,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=4788c4822a4ce08ce43f13bc3e5d2847, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:31,308 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4788c4822a4ce08ce43f13bc3e5d2847, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:31,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-08T21:03:31,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 4788c4822a4ce08ce43f13bc3e5d2847, server=229bab1f9d30,43999,1733691749022 in 190 msec 2024-12-08T21:03:31,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T21:03:31,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=4788c4822a4ce08ce43f13bc3e5d2847, ASSIGN in 354 msec 2024-12-08T21:03:31,316 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:31,316 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691811316"}]},"ts":"1733691811316"} 2024-12-08T21:03:31,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-08T21:03:31,320 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:31,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 805 msec 2024-12-08T21:03:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:03:31,528 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-08T21:03:31,528 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-08T21:03:31,529 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:31,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-08T21:03:31,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:31,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-08T21:03:31,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:31,552 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:31,553 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-08T21:03:31,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T21:03:31,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T21:03:31,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T21:03:31,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-08T21:03:33,008 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-08T21:03:34,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789032' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T21:03:34,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57664 deadline: 1733691815567, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789032' does not exist 2024-12-08T21:03:34,577 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 229bab1f9d30,43999,1733691749022 for closing scanner id = 4749821091608789032 for 4788c4822a4ce08ce43f13bc3e5d2847 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789032' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:03:34,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_993132359_22 at /127.0.0.1:56676 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=185 (was 193), ProcessCount=11 (was 11), AvailableMemoryMB=17001 (was 17002) 2024-12-08T21:03:34,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=185, ProcessCount=11, AvailableMemoryMB=17000 2024-12-08T21:03:34,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-08T21:03:34,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:34,596 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:34,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-08T21:03:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-08T21:03:34,597 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:34,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741847_1023 (size=411) 2024-12-08T21:03:35,012 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21e1e417d6a356c0540aeca8e615876a, NAME => 'testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:35,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741848_1024 (size=62) 2024-12-08T21:03:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-08T21:03:35,429 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:35,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 21e1e417d6a356c0540aeca8e615876a, disabling compactions & flushes 2024-12-08T21:03:35,430 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. after waiting 0 ms 2024-12-08T21:03:35,430 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,430 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,431 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21e1e417d6a356c0540aeca8e615876a: Waiting for close lock at 1733691815430Disabling compacts and flushes for region at 1733691815430Disabling writes for close at 1733691815430Writing region close event to WAL at 1733691815430Closed at 1733691815430 2024-12-08T21:03:35,433 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:35,433 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733691815433"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691815433"}]},"ts":"1733691815433"} 2024-12-08T21:03:35,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:35,439 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:35,440 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691815439"}]},"ts":"1733691815439"} 2024-12-08T21:03:35,443 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-08T21:03:35,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=21e1e417d6a356c0540aeca8e615876a, ASSIGN}] 2024-12-08T21:03:35,445 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=21e1e417d6a356c0540aeca8e615876a, ASSIGN 2024-12-08T21:03:35,446 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=21e1e417d6a356c0540aeca8e615876a, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:35,598 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=21e1e417d6a356c0540aeca8e615876a, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:35,606 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=21e1e417d6a356c0540aeca8e615876a, ASSIGN because future has completed 2024-12-08T21:03:35,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e1e417d6a356c0540aeca8e615876a, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-08T21:03:35,775 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,775 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 21e1e417d6a356c0540aeca8e615876a, NAME => 'testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:35,776 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,776 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:35,776 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,776 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,778 INFO [StoreOpener-21e1e417d6a356c0540aeca8e615876a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,781 INFO [StoreOpener-21e1e417d6a356c0540aeca8e615876a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21e1e417d6a356c0540aeca8e615876a columnFamilyName testFamily 2024-12-08T21:03:35,781 DEBUG [StoreOpener-21e1e417d6a356c0540aeca8e615876a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:35,782 INFO [StoreOpener-21e1e417d6a356c0540aeca8e615876a-1 {}] regionserver.HStore(327): Store=21e1e417d6a356c0540aeca8e615876a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:35,782 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,783 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,784 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,785 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,785 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,788 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,793 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:35,794 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 21e1e417d6a356c0540aeca8e615876a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75079548, jitterRate=0.11877244710922241}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:35,794 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:03:35,795 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 21e1e417d6a356c0540aeca8e615876a: Running coprocessor pre-open hook at 1733691815776Writing region info on filesystem at 1733691815776Initializing all the Stores at 1733691815778 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691815778Cleaning up temporary data from old regions at 1733691815785 (+7 ms)Running coprocessor post-open hooks at 1733691815794 (+9 ms)Region opened successfully at 1733691815795 (+1 ms) 2024-12-08T21:03:35,797 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a., pid=24, masterSystemTime=1733691815762 2024-12-08T21:03:35,800 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,800 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:03:35,800 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=21e1e417d6a356c0540aeca8e615876a, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:35,803 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e1e417d6a356c0540aeca8e615876a, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:35,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T21:03:35,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 21e1e417d6a356c0540aeca8e615876a, server=229bab1f9d30,43999,1733691749022 in 198 msec 2024-12-08T21:03:35,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-08T21:03:35,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=21e1e417d6a356c0540aeca8e615876a, ASSIGN in 364 msec 2024-12-08T21:03:35,810 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:35,811 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691815811"}]},"ts":"1733691815811"} 2024-12-08T21:03:35,813 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-08T21:03:35,814 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:35,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2220 sec 2024-12-08T21:03:36,791 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-08T21:03:37,626 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-08T21:03:37,626 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-08T21:03:37,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:37,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-08T21:03:37,637 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:37,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-08T21:03:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:37,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:37,650 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-08T21:03:37,653 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T21:03:37,655 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T21:03:37,656 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T21:03:37,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-08T21:03:38,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-08T21:03:38,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:38,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-08T21:03:38,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:39,010 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-08T21:03:40,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789036' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T21:03:40,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57664 deadline: 1733691821677, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789036' does not exist 2024-12-08T21:03:40,678 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 229bab1f9d30,43999,1733691749022 for closing scanner id = 4749821091608789036 for 21e1e417d6a356c0540aeca8e615876a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789036' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:03:40,683 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 185), ProcessCount=11 (was 11), AvailableMemoryMB=16991 (was 17000) 2024-12-08T21:03:40,691 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=16991 2024-12-08T21:03:40,693 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:40,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-08T21:03:40,697 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:40,697 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:40,697 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-08T21:03:40,699 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:03:40,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741849_1025 (size=419) 2024-12-08T21:03:41,113 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 901f76f3bed96e80a6bb2be9514fc2bd, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:41,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741850_1026 (size=70) 2024-12-08T21:03:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-08T21:03:41,531 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:41,531 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 901f76f3bed96e80a6bb2be9514fc2bd, disabling compactions & flushes 2024-12-08T21:03:41,531 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,531 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,532 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. after waiting 0 ms 2024-12-08T21:03:41,532 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,532 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,532 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 901f76f3bed96e80a6bb2be9514fc2bd: Waiting for close lock at 1733691821531Disabling compacts and flushes for region at 1733691821531Disabling writes for close at 1733691821532 (+1 ms)Writing region close event to WAL at 1733691821532Closed at 1733691821532 2024-12-08T21:03:41,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:41,535 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733691821535"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691821535"}]},"ts":"1733691821535"} 2024-12-08T21:03:41,537 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:41,539 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:41,539 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691821539"}]},"ts":"1733691821539"} 2024-12-08T21:03:41,541 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-08T21:03:41,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=901f76f3bed96e80a6bb2be9514fc2bd, ASSIGN}] 2024-12-08T21:03:41,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=901f76f3bed96e80a6bb2be9514fc2bd, ASSIGN 2024-12-08T21:03:41,544 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=901f76f3bed96e80a6bb2be9514fc2bd, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:41,695 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=901f76f3bed96e80a6bb2be9514fc2bd, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:41,701 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=901f76f3bed96e80a6bb2be9514fc2bd, ASSIGN because future has completed 2024-12-08T21:03:41,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 901f76f3bed96e80a6bb2be9514fc2bd, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:03:41,866 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 901f76f3bed96e80a6bb2be9514fc2bd, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:41,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:41,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,870 INFO [StoreOpener-901f76f3bed96e80a6bb2be9514fc2bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,872 INFO [StoreOpener-901f76f3bed96e80a6bb2be9514fc2bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 901f76f3bed96e80a6bb2be9514fc2bd columnFamilyName testFamily 2024-12-08T21:03:41,872 DEBUG [StoreOpener-901f76f3bed96e80a6bb2be9514fc2bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:41,873 INFO [StoreOpener-901f76f3bed96e80a6bb2be9514fc2bd-1 {}] regionserver.HStore(327): Store=901f76f3bed96e80a6bb2be9514fc2bd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:41,873 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,875 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,880 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:41,885 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 901f76f3bed96e80a6bb2be9514fc2bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60186847, jitterRate=-0.10314609110355377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:41,885 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:03:41,886 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 901f76f3bed96e80a6bb2be9514fc2bd: Running coprocessor pre-open hook at 1733691821868Writing region info on filesystem at 1733691821868Initializing all the Stores at 1733691821870 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691821870Cleaning up temporary data from old regions at 1733691821877 (+7 ms)Running coprocessor post-open hooks at 1733691821886 (+9 ms)Region opened successfully at 1733691821886 2024-12-08T21:03:41,887 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., pid=27, masterSystemTime=1733691821856 2024-12-08T21:03:41,890 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,890 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:03:41,891 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=901f76f3bed96e80a6bb2be9514fc2bd, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:41,893 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 901f76f3bed96e80a6bb2be9514fc2bd, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:41,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-08T21:03:41,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 901f76f3bed96e80a6bb2be9514fc2bd, server=229bab1f9d30,43999,1733691749022 in 193 msec 2024-12-08T21:03:41,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T21:03:41,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=901f76f3bed96e80a6bb2be9514fc2bd, ASSIGN in 357 msec 2024-12-08T21:03:41,902 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:41,903 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691821902"}]},"ts":"1733691821902"} 2024-12-08T21:03:41,905 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-08T21:03:41,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:41,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2140 sec 2024-12-08T21:03:43,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:03:43,738 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-08T21:03:43,738 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-08T21:03:43,739 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:43,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-08T21:03:43,747 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:43,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-08T21:03:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:43,762 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:43,762 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-08T21:03:43,764 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T21:03:43,766 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-08T21:03:43,867 INFO [master/229bab1f9d30:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T21:03:43,867 INFO [master/229bab1f9d30:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T21:03:45,013 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-08T21:03:47,397 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:48,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-08T21:03:48,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:49,766 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-08T21:03:49,766 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-08T21:03:49,766 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:03:49,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=16979 (was 16991) 2024-12-08T21:03:49,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=144, ProcessCount=11, AvailableMemoryMB=16978 2024-12-08T21:03:49,796 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:49,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-08T21:03:49,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:49,800 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:49,800 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-08T21:03:49,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T21:03:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741851_1027 (size=407) 2024-12-08T21:03:50,218 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a78f1299ac5d678a9497d74dd1cb960d, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741852_1028 (size=58) 2024-12-08T21:03:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-08T21:03:50,631 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:50,631 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing a78f1299ac5d678a9497d74dd1cb960d, disabling compactions & flushes 2024-12-08T21:03:50,631 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,632 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,632 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. after waiting 0 ms 2024-12-08T21:03:50,632 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,632 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,632 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for a78f1299ac5d678a9497d74dd1cb960d: Waiting for close lock at 1733691830631Disabling compacts and flushes for region at 1733691830631Disabling writes for close at 1733691830632 (+1 ms)Writing region close event to WAL at 1733691830632Closed at 1733691830632 2024-12-08T21:03:50,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:50,638 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733691830637"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691830637"}]},"ts":"1733691830637"} 2024-12-08T21:03:50,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:50,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:50,642 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691830642"}]},"ts":"1733691830642"} 2024-12-08T21:03:50,644 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-08T21:03:50,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a78f1299ac5d678a9497d74dd1cb960d, ASSIGN}] 2024-12-08T21:03:50,646 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a78f1299ac5d678a9497d74dd1cb960d, ASSIGN 2024-12-08T21:03:50,647 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a78f1299ac5d678a9497d74dd1cb960d, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:50,799 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a78f1299ac5d678a9497d74dd1cb960d, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:50,804 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a78f1299ac5d678a9497d74dd1cb960d, ASSIGN because future has completed 2024-12-08T21:03:50,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a78f1299ac5d678a9497d74dd1cb960d, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T21:03:50,968 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => a78f1299ac5d678a9497d74dd1cb960d, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:03:50,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:50,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,969 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,970 INFO [StoreOpener-a78f1299ac5d678a9497d74dd1cb960d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,972 INFO [StoreOpener-a78f1299ac5d678a9497d74dd1cb960d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a78f1299ac5d678a9497d74dd1cb960d columnFamilyName testFamily 2024-12-08T21:03:50,972 DEBUG [StoreOpener-a78f1299ac5d678a9497d74dd1cb960d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:50,973 INFO [StoreOpener-a78f1299ac5d678a9497d74dd1cb960d-1 {}] regionserver.HStore(327): Store=a78f1299ac5d678a9497d74dd1cb960d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:03:50,973 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,974 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,974 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,975 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,977 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,980 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:03:50,981 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened a78f1299ac5d678a9497d74dd1cb960d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62645423, jitterRate=-0.06651045382022858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:03:50,981 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:03:50,981 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for a78f1299ac5d678a9497d74dd1cb960d: Running coprocessor pre-open hook at 1733691830969Writing region info on filesystem at 1733691830969Initializing all the Stores at 1733691830970 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691830970Cleaning up temporary data from old regions at 1733691830975 (+5 ms)Running coprocessor post-open hooks at 1733691830981 (+6 ms)Region opened successfully at 1733691830981 2024-12-08T21:03:50,982 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., pid=30, masterSystemTime=1733691830958 2024-12-08T21:03:50,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,984 INFO [RS_OPEN_PRIORITY_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:03:50,985 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a78f1299ac5d678a9497d74dd1cb960d, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:50,987 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a78f1299ac5d678a9497d74dd1cb960d, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:03:50,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T21:03:50,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure a78f1299ac5d678a9497d74dd1cb960d, server=229bab1f9d30,43999,1733691749022 in 183 msec 2024-12-08T21:03:50,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T21:03:50,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=a78f1299ac5d678a9497d74dd1cb960d, ASSIGN in 345 msec 2024-12-08T21:03:50,994 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:03:50,994 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691830994"}]},"ts":"1733691830994"} 2024-12-08T21:03:50,996 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-08T21:03:50,997 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:03:51,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2020 sec 2024-12-08T21:03:51,014 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-08T21:03:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-08T21:03:52,828 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-08T21:03:52,828 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-08T21:03:52,829 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:52,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-08T21:03:52,836 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:03:52,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-08T21:03:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:03:52,849 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:03:52,849 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-08T21:03:52,852 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-08T21:03:52,854 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-08T21:03:52,855 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-08T21:03:52,856 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-08T21:03:56,472 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:03:57,351 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:03:58,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-08T21:03:58,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-08T21:03:58,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789044' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-08T21:03:58,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43999 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:57664 deadline: 1733691844857, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789044' does not exist 2024-12-08T21:03:58,870 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 229bab1f9d30,43999,1733691749022 for closing scanner id = 4749821091608789044 for a78f1299ac5d678a9497d74dd1cb960d of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '4749821091608789044' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:03:58,879 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 144), ProcessCount=11 (was 11), AvailableMemoryMB=16977 (was 16978) 2024-12-08T21:03:58,889 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=16976 2024-12-08T21:03:58,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:03:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-08T21:03:58,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:03:58,893 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:03:58,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-08T21:03:58,894 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:03:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T21:03:58,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741853_1029 (size=423) 2024-12-08T21:03:59,311 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1bc9b7635d08c4a5eda688c3563950a4, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b 2024-12-08T21:03:59,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741854_1030 (size=74) 2024-12-08T21:03:59,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-08T21:03:59,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:03:59,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1bc9b7635d08c4a5eda688c3563950a4, disabling compactions & flushes 2024-12-08T21:03:59,723 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:03:59,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:03:59,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. after waiting 0 ms 2024-12-08T21:03:59,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:03:59,724 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:03:59,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1bc9b7635d08c4a5eda688c3563950a4: Waiting for close lock at 1733691839723Disabling compacts and flushes for region at 1733691839723Disabling writes for close at 1733691839724 (+1 ms)Writing region close event to WAL at 1733691839724Closed at 1733691839724 2024-12-08T21:03:59,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:03:59,727 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733691839727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733691839727"}]},"ts":"1733691839727"} 2024-12-08T21:03:59,730 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:03:59,731 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:03:59,731 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691839731"}]},"ts":"1733691839731"} 2024-12-08T21:03:59,734 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-08T21:03:59,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=1bc9b7635d08c4a5eda688c3563950a4, ASSIGN}] 2024-12-08T21:03:59,736 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=1bc9b7635d08c4a5eda688c3563950a4, ASSIGN 2024-12-08T21:03:59,737 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=1bc9b7635d08c4a5eda688c3563950a4, ASSIGN; state=OFFLINE, location=229bab1f9d30,43999,1733691749022; forceNewPlan=false, retain=false 2024-12-08T21:03:59,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=1bc9b7635d08c4a5eda688c3563950a4, regionState=OPENING, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:03:59,897 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=1bc9b7635d08c4a5eda688c3563950a4, ASSIGN because future has completed 2024-12-08T21:03:59,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9b7635d08c4a5eda688c3563950a4, server=229bab1f9d30,43999,1733691749022}] 2024-12-08T21:03:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T21:04:00,065 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:00,066 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bc9b7635d08c4a5eda688c3563950a4, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:04:00,066 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,066 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:04:00,066 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,066 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,068 INFO [StoreOpener-1bc9b7635d08c4a5eda688c3563950a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,071 INFO [StoreOpener-1bc9b7635d08c4a5eda688c3563950a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bc9b7635d08c4a5eda688c3563950a4 columnFamilyName testFamily 2024-12-08T21:04:00,071 DEBUG [StoreOpener-1bc9b7635d08c4a5eda688c3563950a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:04:00,071 INFO [StoreOpener-1bc9b7635d08c4a5eda688c3563950a4-1 {}] regionserver.HStore(327): Store=1bc9b7635d08c4a5eda688c3563950a4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:04:00,072 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,072 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,073 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,074 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,074 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,076 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,079 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:04:00,080 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 1bc9b7635d08c4a5eda688c3563950a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67477237, jitterRate=0.005489185452461243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T21:04:00,080 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:00,081 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 1bc9b7635d08c4a5eda688c3563950a4: Running coprocessor pre-open hook at 1733691840066Writing region info on filesystem at 1733691840067 (+1 ms)Initializing all the Stores at 1733691840068 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733691840068Cleaning up temporary data from old regions at 1733691840074 (+6 ms)Running coprocessor post-open hooks at 1733691840080 (+6 ms)Region opened successfully at 1733691840081 (+1 ms) 2024-12-08T21:04:00,125 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., pid=33, masterSystemTime=1733691840053 2024-12-08T21:04:00,128 DEBUG [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:00,128 INFO [RS_OPEN_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:00,129 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=1bc9b7635d08c4a5eda688c3563950a4, regionState=OPEN, openSeqNum=2, regionLocation=229bab1f9d30,43999,1733691749022 2024-12-08T21:04:00,133 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bc9b7635d08c4a5eda688c3563950a4, server=229bab1f9d30,43999,1733691749022 because future has completed 2024-12-08T21:04:00,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T21:04:00,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bc9b7635d08c4a5eda688c3563950a4, server=229bab1f9d30,43999,1733691749022 in 238 msec 2024-12-08T21:04:00,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-08T21:04:00,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=1bc9b7635d08c4a5eda688c3563950a4, ASSIGN in 404 msec 2024-12-08T21:04:00,141 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:04:00,142 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733691840141"}]},"ts":"1733691840141"} 2024-12-08T21:04:00,144 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-08T21:04:00,145 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:04:00,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2540 sec 2024-12-08T21:04:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41621 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T21:04:01,925 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-08T21:04:01,925 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-08T21:04:01,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:04:01,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-08T21:04:01,933 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T21:04:01,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-08T21:04:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:04:01,947 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-08T21:04:01,947 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-08T21:04:01,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-08T21:04:01,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T21:04:02,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-08T21:04:02,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-08T21:04:02,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:04:03,017 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-08T21:04:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-08T21:04:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2] 2024-12-08T21:04:03,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43999 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-08T21:04:04,302 DEBUG [master/229bab1f9d30:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T21:04:04,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=229bab1f9d30:43999 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=977ms 2024-12-08T21:04:04,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=977ms 2024-12-08T21:04:04,956 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., hostname=229bab1f9d30,43999,1733691749022, seqNum=2 from cache 2024-12-08T21:04:04,971 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=112 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=16962 (was 16976) 2024-12-08T21:04:04,972 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:04:04,972 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:04:04,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:04,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:04,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:04:04,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T21:04:04,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:04:04,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:04:04,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:04,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:04,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:04:04,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T21:04:04,978 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=331909808, stopped=false 2024-12-08T21:04:04,978 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=229bab1f9d30,41621,1733691748281 2024-12-08T21:04:05,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:04:05,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:04:05,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:04:05,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:04:05,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:04:05,029 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:04:05,030 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:04:05,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:04:05,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:05,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:04:05,031 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '229bab1f9d30,43999,1733691749022' ***** 2024-12-08T21:04:05,031 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:04:05,032 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:04:05,032 INFO [RS:0;229bab1f9d30:43999 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:04:05,032 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:04:05,033 INFO [RS:0;229bab1f9d30:43999 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:04:05,033 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 38588e6d1cec3f8d460fda2df0ad9f8d 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 21e1e417d6a356c0540aeca8e615876a 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a507a1df86c7c06aed001d2290ba682 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 901f76f3bed96e80a6bb2be9514fc2bd 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 1bc9b7635d08c4a5eda688c3563950a4 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 4788c4822a4ce08ce43f13bc3e5d2847 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 319e5659b18dc5e2899c9e019b76dea1 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:05,034 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38588e6d1cec3f8d460fda2df0ad9f8d, disabling compactions & flushes 2024-12-08T21:04:05,034 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:04:05,034 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(959): stopping server 229bab1f9d30,43999,1733691749022 2024-12-08T21:04:05,035 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:04:05,035 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. after waiting 0 ms 2024-12-08T21:04:05,035 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;229bab1f9d30:43999. 2024-12-08T21:04:05,035 DEBUG [RS:0;229bab1f9d30:43999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:04:05,035 DEBUG [RS:0;229bab1f9d30:43999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:04:05,035 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:04:05,036 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T21:04:05,036 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-08T21:04:05,036 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:04:05,036 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:04:05,036 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:04:05,036 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:04:05,037 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 21e1e417d6a356c0540aeca8e615876a, 319e5659b18dc5e2899c9e019b76dea1, 38588e6d1cec3f8d460fda2df0ad9f8d, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,037 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:04:05,039 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-08T21:04:05,039 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 38588e6d1cec3f8d460fda2df0ad9f8d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:05,087 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/.tmp/testFamily/e173650390e5449c9e2e75651bb2ab7d is 61, key is row-0/testFamily:testQualifier/1733691797254/Put/seqid=0 2024-12-08T21:04:05,092 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/info/ead5888451344b82bf30f459b10871da is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731./info:regioninfo/1733691784757/Put/seqid=0 2024-12-08T21:04:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741856_1032 (size=20564) 2024-12-08T21:04:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741855_1031 (size=5049) 2024-12-08T21:04:05,237 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 21e1e417d6a356c0540aeca8e615876a, 319e5659b18dc5e2899c9e019b76dea1, 38588e6d1cec3f8d460fda2df0ad9f8d, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,437 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 21e1e417d6a356c0540aeca8e615876a, 319e5659b18dc5e2899c9e019b76dea1, 38588e6d1cec3f8d460fda2df0ad9f8d, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,477 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-08T21:04:05,504 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/.tmp/testFamily/e173650390e5449c9e2e75651bb2ab7d 2024-12-08T21:04:05,505 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/info/ead5888451344b82bf30f459b10871da 2024-12-08T21:04:05,548 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/.tmp/testFamily/e173650390e5449c9e2e75651bb2ab7d as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/testFamily/e173650390e5449c9e2e75651bb2ab7d 2024-12-08T21:04:05,557 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/testFamily/e173650390e5449c9e2e75651bb2ab7d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:05,562 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/ns/48c8e8ac1e524c9aaef73473fbd955bf is 43, key is default/ns:d/1733691783639/Put/seqid=0 2024-12-08T21:04:05,564 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38588e6d1cec3f8d460fda2df0ad9f8d in 525ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:05,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741857_1033 (size=5153) 2024-12-08T21:04:05,568 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:04:05,570 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNextAsync/38588e6d1cec3f8d460fda2df0ad9f8d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:05,573 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:04:05,573 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38588e6d1cec3f8d460fda2df0ad9f8d: Waiting for close lock at 1733691845034Running coprocessor pre-close hooks at 1733691845034Disabling compacts and flushes for region at 1733691845034Disabling writes for close at 1733691845035 (+1 ms)Obtaining lock to block concurrent updates at 1733691845040 (+5 ms)Preparing flush snapshotting stores in 38588e6d1cec3f8d460fda2df0ad9f8d at 1733691845040Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691845047 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. at 1733691845048 (+1 ms)Flushing 38588e6d1cec3f8d460fda2df0ad9f8d/testFamily: creating writer at 1733691845049 (+1 ms)Flushing 38588e6d1cec3f8d460fda2df0ad9f8d/testFamily: appending metadata at 1733691845083 (+34 ms)Flushing 38588e6d1cec3f8d460fda2df0ad9f8d/testFamily: closing flushed file at 1733691845085 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49991f36: reopening flushed file at 1733691845546 (+461 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 38588e6d1cec3f8d460fda2df0ad9f8d in 525ms, sequenceid=8, compaction requested=false at 1733691845564 (+18 ms)Writing region close event to WAL at 1733691845566 (+2 ms)Running coprocessor post-close hooks at 1733691845571 (+5 ms)Closed at 1733691845573 (+2 ms) 2024-12-08T21:04:05,574 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733691794187.38588e6d1cec3f8d460fda2df0ad9f8d. 2024-12-08T21:04:05,574 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 21e1e417d6a356c0540aeca8e615876a, disabling compactions & flushes 2024-12-08T21:04:05,574 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:04:05,574 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:04:05,574 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. after waiting 0 ms 2024-12-08T21:04:05,574 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:04:05,574 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 21e1e417d6a356c0540aeca8e615876a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:05,596 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/.tmp/testFamily/f272833e30884b67bc5340a4b4cbd19c is 61, key is row-0/testFamily:testQualifier/1733691817642/Put/seqid=0 2024-12-08T21:04:05,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741858_1034 (size=5049) 2024-12-08T21:04:05,638 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 21e1e417d6a356c0540aeca8e615876a, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,838 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 21e1e417d6a356c0540aeca8e615876a, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:05,928 INFO [regionserver/229bab1f9d30:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T21:04:05,929 INFO [regionserver/229bab1f9d30:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T21:04:05,968 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/ns/48c8e8ac1e524c9aaef73473fbd955bf 2024-12-08T21:04:05,998 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/table/bec269d45ab1499d80bc230ea8621bc2 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733691784773/Put/seqid=0 2024-12-08T21:04:06,002 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/.tmp/testFamily/f272833e30884b67bc5340a4b4cbd19c 2024-12-08T21:04:06,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741859_1035 (size=6699) 2024-12-08T21:04:06,010 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/.tmp/testFamily/f272833e30884b67bc5340a4b4cbd19c as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/testFamily/f272833e30884b67bc5340a4b4cbd19c 2024-12-08T21:04:06,018 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/testFamily/f272833e30884b67bc5340a4b4cbd19c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:06,019 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 21e1e417d6a356c0540aeca8e615876a in 445ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:06,024 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNext/21e1e417d6a356c0540aeca8e615876a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:06,025 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:04:06,025 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 21e1e417d6a356c0540aeca8e615876a: Waiting for close lock at 1733691845574Running coprocessor pre-close hooks at 1733691845574Disabling compacts and flushes for region at 1733691845574Disabling writes for close at 1733691845574Obtaining lock to block concurrent updates at 1733691845574Preparing flush snapshotting stores in 21e1e417d6a356c0540aeca8e615876a at 1733691845574Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691845575 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. at 1733691845575Flushing 21e1e417d6a356c0540aeca8e615876a/testFamily: creating writer at 1733691845575Flushing 21e1e417d6a356c0540aeca8e615876a/testFamily: appending metadata at 1733691845595 (+20 ms)Flushing 21e1e417d6a356c0540aeca8e615876a/testFamily: closing flushed file at 1733691845595Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@363ea25e: reopening flushed file at 1733691846009 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 21e1e417d6a356c0540aeca8e615876a in 445ms, sequenceid=8, compaction requested=false at 1733691846019 (+10 ms)Writing region close event to WAL at 1733691846020 (+1 ms)Running coprocessor post-close hooks at 1733691846025 (+5 ms)Closed at 1733691846025 2024-12-08T21:04:06,025 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733691814592.21e1e417d6a356c0540aeca8e615876a. 2024-12-08T21:04:06,025 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a507a1df86c7c06aed001d2290ba682, disabling compactions & flushes 2024-12-08T21:04:06,025 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:04:06,026 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:04:06,026 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. after waiting 0 ms 2024-12-08T21:04:06,026 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:04:06,026 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a507a1df86c7c06aed001d2290ba682 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:06,038 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-08T21:04:06,038 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1325): Online Regions={1a507a1df86c7c06aed001d2290ba682=testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682., 901f76f3bed96e80a6bb2be9514fc2bd=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., 1bc9b7635d08c4a5eda688c3563950a4=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., d301d980e330faad26a3ed42cc20b3c5=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., 4788c4822a4ce08ce43f13bc3e5d2847=testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., 1588230740=hbase:meta,,1.1588230740, 319e5659b18dc5e2899c9e019b76dea1=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., a78f1299ac5d678a9497d74dd1cb960d=hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., 879a1685c7f2dd2f7b50136f3562a731=testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.} 2024-12-08T21:04:06,039 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:06,041 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/.tmp/testFamily/bf13b17ece434cd4a2a6881069a73209 is 61, key is row-0/testFamily:testQualifier/1733691804341/Put/seqid=0 2024-12-08T21:04:06,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741860_1036 (size=5049) 2024-12-08T21:04:06,239 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:06,406 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/table/bec269d45ab1499d80bc230ea8621bc2 2024-12-08T21:04:06,421 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/info/ead5888451344b82bf30f459b10871da as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/info/ead5888451344b82bf30f459b10871da 2024-12-08T21:04:06,429 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/info/ead5888451344b82bf30f459b10871da, entries=100, sequenceid=56, filesize=20.1 K 2024-12-08T21:04:06,430 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/ns/48c8e8ac1e524c9aaef73473fbd955bf as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/ns/48c8e8ac1e524c9aaef73473fbd955bf 2024-12-08T21:04:06,438 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/ns/48c8e8ac1e524c9aaef73473fbd955bf, entries=2, sequenceid=56, filesize=5.0 K 2024-12-08T21:04:06,439 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/.tmp/table/bec269d45ab1499d80bc230ea8621bc2 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/table/bec269d45ab1499d80bc230ea8621bc2 2024-12-08T21:04:06,439 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a507a1df86c7c06aed001d2290ba682, 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:06,446 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/.tmp/testFamily/bf13b17ece434cd4a2a6881069a73209 2024-12-08T21:04:06,446 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/table/bec269d45ab1499d80bc230ea8621bc2, entries=20, sequenceid=56, filesize=6.5 K 2024-12-08T21:04:06,448 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false 2024-12-08T21:04:06,453 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-08T21:04:06,454 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:04:06,454 INFO [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:04:06,455 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733691845036Running coprocessor pre-close hooks at 1733691845036Disabling compacts and flushes for region at 1733691845036Disabling writes for close at 1733691845037 (+1 ms)Obtaining lock to block concurrent updates at 1733691845040 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733691845040Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733691845047 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733691845048 (+1 ms)Flushing 1588230740/info: creating writer at 1733691845049 (+1 ms)Flushing 1588230740/info: appending metadata at 1733691845089 (+40 ms)Flushing 1588230740/info: closing flushed file at 1733691845089Flushing 1588230740/ns: creating writer at 1733691845546 (+457 ms)Flushing 1588230740/ns: appending metadata at 1733691845561 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733691845561Flushing 1588230740/table: creating writer at 1733691845982 (+421 ms)Flushing 1588230740/table: appending metadata at 1733691845998 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733691845998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c90c88a: reopening flushed file at 1733691846420 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7503051b: reopening flushed file at 1733691846429 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d7613de: reopening flushed file at 1733691846438 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1411ms, sequenceid=56, compaction requested=false at 1733691846448 (+10 ms)Writing region close event to WAL at 1733691846449 (+1 ms)Running coprocessor post-close hooks at 1733691846454 (+5 ms)Closed at 1733691846454 2024-12-08T21:04:06,455 DEBUG [RS_CLOSE_META-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T21:04:06,455 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/.tmp/testFamily/bf13b17ece434cd4a2a6881069a73209 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/testFamily/bf13b17ece434cd4a2a6881069a73209 2024-12-08T21:04:06,462 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/testFamily/bf13b17ece434cd4a2a6881069a73209, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:06,463 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a507a1df86c7c06aed001d2290ba682 in 436ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:06,467 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextException/1a507a1df86c7c06aed001d2290ba682/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:06,468 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a507a1df86c7c06aed001d2290ba682: Waiting for close lock at 1733691846025Running coprocessor pre-close hooks at 1733691846025Disabling compacts and flushes for region at 1733691846025Disabling writes for close at 1733691846026 (+1 ms)Obtaining lock to block concurrent updates at 1733691846026Preparing flush snapshotting stores in 1a507a1df86c7c06aed001d2290ba682 at 1733691846026Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691846026Flushing stores of testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. at 1733691846027 (+1 ms)Flushing 1a507a1df86c7c06aed001d2290ba682/testFamily: creating writer at 1733691846027Flushing 1a507a1df86c7c06aed001d2290ba682/testFamily: appending metadata at 1733691846041 (+14 ms)Flushing 1a507a1df86c7c06aed001d2290ba682/testFamily: closing flushed file at 1733691846041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dae4edf: reopening flushed file at 1733691846454 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a507a1df86c7c06aed001d2290ba682 in 436ms, sequenceid=8, compaction requested=false at 1733691846463 (+9 ms)Writing region close event to WAL at 1733691846463Running coprocessor post-close hooks at 1733691846468 (+5 ms)Closed at 1733691846468 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733691803303.1a507a1df86c7c06aed001d2290ba682. 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 901f76f3bed96e80a6bb2be9514fc2bd, disabling compactions & flushes 2024-12-08T21:04:06,468 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. after waiting 0 ms 2024-12-08T21:04:06,468 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:04:06,468 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 901f76f3bed96e80a6bb2be9514fc2bd 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:06,492 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/.tmp/testFamily/7f9bb7824ff4406f8ff13aeea0c49554 is 61, key is row-0/testFamily:testQualifier/1733691823754/Put/seqid=0 2024-12-08T21:04:06,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741861_1037 (size=5049) 2024-12-08T21:04:06,640 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:06,840 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, 901f76f3bed96e80a6bb2be9514fc2bd, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:06,898 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/.tmp/testFamily/7f9bb7824ff4406f8ff13aeea0c49554 2024-12-08T21:04:06,913 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/.tmp/testFamily/7f9bb7824ff4406f8ff13aeea0c49554 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/testFamily/7f9bb7824ff4406f8ff13aeea0c49554 2024-12-08T21:04:06,921 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/testFamily/7f9bb7824ff4406f8ff13aeea0c49554, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:06,922 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 901f76f3bed96e80a6bb2be9514fc2bd in 454ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:06,929 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/901f76f3bed96e80a6bb2be9514fc2bd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:06,929 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:04:06,929 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 901f76f3bed96e80a6bb2be9514fc2bd: Waiting for close lock at 1733691846468Running coprocessor pre-close hooks at 1733691846468Disabling compacts and flushes for region at 1733691846468Disabling writes for close at 1733691846468Obtaining lock to block concurrent updates at 1733691846468Preparing flush snapshotting stores in 901f76f3bed96e80a6bb2be9514fc2bd at 1733691846468Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691846469 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. at 1733691846469Flushing 901f76f3bed96e80a6bb2be9514fc2bd/testFamily: creating writer at 1733691846469Flushing 901f76f3bed96e80a6bb2be9514fc2bd/testFamily: appending metadata at 1733691846492 (+23 ms)Flushing 901f76f3bed96e80a6bb2be9514fc2bd/testFamily: closing flushed file at 1733691846492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c4f8675: reopening flushed file at 1733691846911 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 901f76f3bed96e80a6bb2be9514fc2bd in 454ms, sequenceid=8, compaction requested=false at 1733691846922 (+11 ms)Writing region close event to WAL at 1733691846923 (+1 ms)Running coprocessor post-close hooks at 1733691846929 (+6 ms)Closed at 1733691846929 2024-12-08T21:04:06,930 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733691820693.901f76f3bed96e80a6bb2be9514fc2bd. 2024-12-08T21:04:06,930 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1bc9b7635d08c4a5eda688c3563950a4, disabling compactions & flushes 2024-12-08T21:04:06,930 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:06,930 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:06,930 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. after waiting 0 ms 2024-12-08T21:04:06,930 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:06,930 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1bc9b7635d08c4a5eda688c3563950a4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:06,947 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/.tmp/testFamily/5a701cfc95e443ab9e03e7d9512fcaeb is 61, key is row-0/testFamily:testQualifier/1733691841939/Put/seqid=0 2024-12-08T21:04:06,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741862_1038 (size=5049) 2024-12-08T21:04:07,040 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-08T21:04:07,040 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1325): Online Regions={1bc9b7635d08c4a5eda688c3563950a4=testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., d301d980e330faad26a3ed42cc20b3c5=hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., 4788c4822a4ce08ce43f13bc3e5d2847=testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., 319e5659b18dc5e2899c9e019b76dea1=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., a78f1299ac5d678a9497d74dd1cb960d=hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., 879a1685c7f2dd2f7b50136f3562a731=testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.} 2024-12-08T21:04:07,041 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:07,241 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 1bc9b7635d08c4a5eda688c3563950a4, 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:07,353 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/.tmp/testFamily/5a701cfc95e443ab9e03e7d9512fcaeb 2024-12-08T21:04:07,364 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/.tmp/testFamily/5a701cfc95e443ab9e03e7d9512fcaeb as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/testFamily/5a701cfc95e443ab9e03e7d9512fcaeb 2024-12-08T21:04:07,374 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/testFamily/5a701cfc95e443ab9e03e7d9512fcaeb, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:07,375 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1bc9b7635d08c4a5eda688c3563950a4 in 445ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:07,380 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScannerAsync/1bc9b7635d08c4a5eda688c3563950a4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:07,381 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1bc9b7635d08c4a5eda688c3563950a4: Waiting for close lock at 1733691846930Running coprocessor pre-close hooks at 1733691846930Disabling compacts and flushes for region at 1733691846930Disabling writes for close at 1733691846930Obtaining lock to block concurrent updates at 1733691846930Preparing flush snapshotting stores in 1bc9b7635d08c4a5eda688c3563950a4 at 1733691846930Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691846930Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. at 1733691846931 (+1 ms)Flushing 1bc9b7635d08c4a5eda688c3563950a4/testFamily: creating writer at 1733691846931Flushing 1bc9b7635d08c4a5eda688c3563950a4/testFamily: appending metadata at 1733691846947 (+16 ms)Flushing 1bc9b7635d08c4a5eda688c3563950a4/testFamily: closing flushed file at 1733691846947Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71dc720e: reopening flushed file at 1733691847362 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1bc9b7635d08c4a5eda688c3563950a4 in 445ms, sequenceid=8, compaction requested=false at 1733691847375 (+13 ms)Writing region close event to WAL at 1733691847376 (+1 ms)Running coprocessor post-close hooks at 1733691847381 (+5 ms)Closed at 1733691847381 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733691838890.1bc9b7635d08c4a5eda688c3563950a4. 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d301d980e330faad26a3ed42cc20b3c5, disabling compactions & flushes 2024-12-08T21:04:07,381 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. after waiting 0 ms 2024-12-08T21:04:07,381 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:04:07,381 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d301d980e330faad26a3ed42cc20b3c5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:07,395 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/.tmp/testFamily/3b9ad9b991ff4816a7aacf93a3702730 is 61, key is row-0/testFamily:testQualifier/1733691788132/Put/seqid=0 2024-12-08T21:04:07,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741863_1039 (size=5049) 2024-12-08T21:04:07,441 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:07,642 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d, d301d980e330faad26a3ed42cc20b3c5 2024-12-08T21:04:07,803 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/.tmp/testFamily/3b9ad9b991ff4816a7aacf93a3702730 2024-12-08T21:04:07,818 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/.tmp/testFamily/3b9ad9b991ff4816a7aacf93a3702730 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/testFamily/3b9ad9b991ff4816a7aacf93a3702730 2024-12-08T21:04:07,827 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/testFamily/3b9ad9b991ff4816a7aacf93a3702730, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:07,828 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d301d980e330faad26a3ed42cc20b3c5 in 447ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:07,832 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnOpenScanner/d301d980e330faad26a3ed42cc20b3c5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:07,833 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:04:07,833 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d301d980e330faad26a3ed42cc20b3c5: Waiting for close lock at 1733691847381Running coprocessor pre-close hooks at 1733691847381Disabling compacts and flushes for region at 1733691847381Disabling writes for close at 1733691847381Obtaining lock to block concurrent updates at 1733691847381Preparing flush snapshotting stores in d301d980e330faad26a3ed42cc20b3c5 at 1733691847381Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691847382 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. at 1733691847382Flushing d301d980e330faad26a3ed42cc20b3c5/testFamily: creating writer at 1733691847382Flushing d301d980e330faad26a3ed42cc20b3c5/testFamily: appending metadata at 1733691847395 (+13 ms)Flushing d301d980e330faad26a3ed42cc20b3c5/testFamily: closing flushed file at 1733691847395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29a02fdd: reopening flushed file at 1733691847816 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d301d980e330faad26a3ed42cc20b3c5 in 447ms, sequenceid=8, compaction requested=false at 1733691847828 (+12 ms)Writing region close event to WAL at 1733691847828Running coprocessor post-close hooks at 1733691847833 (+5 ms)Closed at 1733691847833 2024-12-08T21:04:07,834 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733691785074.d301d980e330faad26a3ed42cc20b3c5. 2024-12-08T21:04:07,834 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4788c4822a4ce08ce43f13bc3e5d2847, disabling compactions & flushes 2024-12-08T21:04:07,834 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:04:07,834 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:04:07,834 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. after waiting 0 ms 2024-12-08T21:04:07,834 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:04:07,834 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4788c4822a4ce08ce43f13bc3e5d2847 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:07,842 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:07,850 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/.tmp/testFamily/2a9943d48dc147939dbbde8fe0d25625 is 61, key is row-0/testFamily:testQualifier/1733691811543/Put/seqid=0 2024-12-08T21:04:07,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741864_1040 (size=5049) 2024-12-08T21:04:08,042 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-08T21:04:08,043 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1325): Online Regions={4788c4822a4ce08ce43f13bc3e5d2847=testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., 319e5659b18dc5e2899c9e019b76dea1=testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., a78f1299ac5d678a9497d74dd1cb960d=hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., 879a1685c7f2dd2f7b50136f3562a731=testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.} 2024-12-08T21:04:08,043 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:08,243 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 4788c4822a4ce08ce43f13bc3e5d2847, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:08,257 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/.tmp/testFamily/2a9943d48dc147939dbbde8fe0d25625 2024-12-08T21:04:08,268 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/.tmp/testFamily/2a9943d48dc147939dbbde8fe0d25625 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/testFamily/2a9943d48dc147939dbbde8fe0d25625 2024-12-08T21:04:08,275 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/testFamily/2a9943d48dc147939dbbde8fe0d25625, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:08,276 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4788c4822a4ce08ce43f13bc3e5d2847 in 442ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:08,281 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnNextAsync/4788c4822a4ce08ce43f13bc3e5d2847/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:08,282 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4788c4822a4ce08ce43f13bc3e5d2847: Waiting for close lock at 1733691847834Running coprocessor pre-close hooks at 1733691847834Disabling compacts and flushes for region at 1733691847834Disabling writes for close at 1733691847834Obtaining lock to block concurrent updates at 1733691847834Preparing flush snapshotting stores in 4788c4822a4ce08ce43f13bc3e5d2847 at 1733691847834Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691847834Flushing stores of testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. at 1733691847835 (+1 ms)Flushing 4788c4822a4ce08ce43f13bc3e5d2847/testFamily: creating writer at 1733691847835Flushing 4788c4822a4ce08ce43f13bc3e5d2847/testFamily: appending metadata at 1733691847850 (+15 ms)Flushing 4788c4822a4ce08ce43f13bc3e5d2847/testFamily: closing flushed file at 1733691847850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d9cb30a: reopening flushed file at 1733691848267 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4788c4822a4ce08ce43f13bc3e5d2847 in 442ms, sequenceid=8, compaction requested=false at 1733691848276 (+9 ms)Writing region close event to WAL at 1733691848277 (+1 ms)Running coprocessor post-close hooks at 1733691848282 (+5 ms)Closed at 1733691848282 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733691810514.4788c4822a4ce08ce43f13bc3e5d2847. 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 319e5659b18dc5e2899c9e019b76dea1, disabling compactions & flushes 2024-12-08T21:04:08,282 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. after waiting 0 ms 2024-12-08T21:04:08,282 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:04:08,283 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 319e5659b18dc5e2899c9e019b76dea1 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:08,298 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/.tmp/testFamily/50b7f7a301394e8e996324cb3057427d is 61, key is row-0/testFamily:testQualifier/1733691807464/Put/seqid=0 2024-12-08T21:04:08,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741865_1041 (size=5049) 2024-12-08T21:04:08,444 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:08,644 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 319e5659b18dc5e2899c9e019b76dea1, 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:08,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:04:08,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T21:04:08,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-08T21:04:08,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-08T21:04:08,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-08T21:04:08,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-08T21:04:08,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-08T21:04:08,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-08T21:04:08,705 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/.tmp/testFamily/50b7f7a301394e8e996324cb3057427d 2024-12-08T21:04:08,719 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/.tmp/testFamily/50b7f7a301394e8e996324cb3057427d as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/testFamily/50b7f7a301394e8e996324cb3057427d 2024-12-08T21:04:08,726 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/testFamily/50b7f7a301394e8e996324cb3057427d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:08,727 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 319e5659b18dc5e2899c9e019b76dea1 in 444ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:08,732 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testNormalScanTimeoutOnOpenScanner/319e5659b18dc5e2899c9e019b76dea1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:08,732 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:04:08,732 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 319e5659b18dc5e2899c9e019b76dea1: Waiting for close lock at 1733691848282Running coprocessor pre-close hooks at 1733691848282Disabling compacts and flushes for region at 1733691848282Disabling writes for close at 1733691848282Obtaining lock to block concurrent updates at 1733691848283 (+1 ms)Preparing flush snapshotting stores in 319e5659b18dc5e2899c9e019b76dea1 at 1733691848283Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691848283Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. at 1733691848284 (+1 ms)Flushing 319e5659b18dc5e2899c9e019b76dea1/testFamily: creating writer at 1733691848284Flushing 319e5659b18dc5e2899c9e019b76dea1/testFamily: appending metadata at 1733691848298 (+14 ms)Flushing 319e5659b18dc5e2899c9e019b76dea1/testFamily: closing flushed file at 1733691848298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@634a65ed: reopening flushed file at 1733691848718 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 319e5659b18dc5e2899c9e019b76dea1 in 444ms, sequenceid=8, compaction requested=false at 1733691848727 (+9 ms)Writing region close event to WAL at 1733691848728 (+1 ms)Running coprocessor post-close hooks at 1733691848732 (+4 ms)Closed at 1733691848732 2024-12-08T21:04:08,732 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733691804399.319e5659b18dc5e2899c9e019b76dea1. 2024-12-08T21:04:08,733 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a78f1299ac5d678a9497d74dd1cb960d, disabling compactions & flushes 2024-12-08T21:04:08,733 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:04:08,733 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:04:08,733 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. after waiting 0 ms 2024-12-08T21:04:08,733 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:04:08,733 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a78f1299ac5d678a9497d74dd1cb960d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:08,748 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/.tmp/testFamily/3778205bcdb04aba9b24bb3a2e0d9d3d is 61, key is row-0/testFamily:testQualifier/1733691832841/Put/seqid=0 2024-12-08T21:04:08,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741866_1042 (size=5049) 2024-12-08T21:04:08,845 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:09,045 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T21:04:09,045 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1325): Online Regions={a78f1299ac5d678a9497d74dd1cb960d=hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., 879a1685c7f2dd2f7b50136f3562a731=testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731.} 2024-12-08T21:04:09,045 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 879a1685c7f2dd2f7b50136f3562a731, a78f1299ac5d678a9497d74dd1cb960d 2024-12-08T21:04:09,155 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/.tmp/testFamily/3778205bcdb04aba9b24bb3a2e0d9d3d 2024-12-08T21:04:09,168 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/.tmp/testFamily/3778205bcdb04aba9b24bb3a2e0d9d3d as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/testFamily/3778205bcdb04aba9b24bb3a2e0d9d3d 2024-12-08T21:04:09,177 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/testFamily/3778205bcdb04aba9b24bb3a2e0d9d3d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:09,178 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a78f1299ac5d678a9497d74dd1cb960d in 445ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:09,182 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/hbase/testMetaScanTimeoutOnNext/a78f1299ac5d678a9497d74dd1cb960d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:09,183 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:04:09,183 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a78f1299ac5d678a9497d74dd1cb960d: Waiting for close lock at 1733691848732Running coprocessor pre-close hooks at 1733691848732Disabling compacts and flushes for region at 1733691848733 (+1 ms)Disabling writes for close at 1733691848733Obtaining lock to block concurrent updates at 1733691848733Preparing flush snapshotting stores in a78f1299ac5d678a9497d74dd1cb960d at 1733691848733Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691848733Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. at 1733691848734 (+1 ms)Flushing a78f1299ac5d678a9497d74dd1cb960d/testFamily: creating writer at 1733691848734Flushing a78f1299ac5d678a9497d74dd1cb960d/testFamily: appending metadata at 1733691848748 (+14 ms)Flushing a78f1299ac5d678a9497d74dd1cb960d/testFamily: closing flushed file at 1733691848748Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@630b3f86: reopening flushed file at 1733691849167 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a78f1299ac5d678a9497d74dd1cb960d in 445ms, sequenceid=8, compaction requested=false at 1733691849178 (+11 ms)Writing region close event to WAL at 1733691849179 (+1 ms)Running coprocessor post-close hooks at 1733691849183 (+4 ms)Closed at 1733691849183 2024-12-08T21:04:09,183 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733691829796.a78f1299ac5d678a9497d74dd1cb960d. 2024-12-08T21:04:09,184 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 879a1685c7f2dd2f7b50136f3562a731, disabling compactions & flushes 2024-12-08T21:04:09,184 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:04:09,184 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:04:09,184 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. after waiting 0 ms 2024-12-08T21:04:09,184 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:04:09,184 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 879a1685c7f2dd2f7b50136f3562a731 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-08T21:04:09,198 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/.tmp/testFamily/98ef281a0ed14ccbb5e92ac094f320ef is 61, key is row-0/testFamily:testQualifier/1733691784985/Put/seqid=0 2024-12-08T21:04:09,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741867_1043 (size=5049) 2024-12-08T21:04:09,245 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:04:09,446 DEBUG [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1351): Waiting on 879a1685c7f2dd2f7b50136f3562a731 2024-12-08T21:04:09,606 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/.tmp/testFamily/98ef281a0ed14ccbb5e92ac094f320ef 2024-12-08T21:04:09,620 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/.tmp/testFamily/98ef281a0ed14ccbb5e92ac094f320ef as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/testFamily/98ef281a0ed14ccbb5e92ac094f320ef 2024-12-08T21:04:09,627 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/testFamily/98ef281a0ed14ccbb5e92ac094f320ef, entries=4, sequenceid=8, filesize=4.9 K 2024-12-08T21:04:09,628 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 879a1685c7f2dd2f7b50136f3562a731 in 444ms, sequenceid=8, compaction requested=false 2024-12-08T21:04:09,632 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/data/default/testRetryOutOfOrderScannerNextExceptionAsync/879a1685c7f2dd2f7b50136f3562a731/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-08T21:04:09,633 INFO [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:04:09,633 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 879a1685c7f2dd2f7b50136f3562a731: Waiting for close lock at 1733691849184Running coprocessor pre-close hooks at 1733691849184Disabling compacts and flushes for region at 1733691849184Disabling writes for close at 1733691849184Obtaining lock to block concurrent updates at 1733691849184Preparing flush snapshotting stores in 879a1685c7f2dd2f7b50136f3562a731 at 1733691849184Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733691849184Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. at 1733691849185 (+1 ms)Flushing 879a1685c7f2dd2f7b50136f3562a731/testFamily: creating writer at 1733691849185Flushing 879a1685c7f2dd2f7b50136f3562a731/testFamily: appending metadata at 1733691849198 (+13 ms)Flushing 879a1685c7f2dd2f7b50136f3562a731/testFamily: closing flushed file at 1733691849198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77718d0a: reopening flushed file at 1733691849619 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 879a1685c7f2dd2f7b50136f3562a731 in 444ms, sequenceid=8, compaction requested=false at 1733691849628 (+9 ms)Writing region close event to WAL at 1733691849628Running coprocessor post-close hooks at 1733691849633 (+5 ms)Closed at 1733691849633 2024-12-08T21:04:09,633 DEBUG [RS_CLOSE_REGION-regionserver/229bab1f9d30:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733691783905.879a1685c7f2dd2f7b50136f3562a731. 2024-12-08T21:04:09,646 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(976): stopping server 229bab1f9d30,43999,1733691749022; all regions closed. 2024-12-08T21:04:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741834_1010 (size=19479) 2024-12-08T21:04:09,660 DEBUG [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/oldWALs 2024-12-08T21:04:09,660 INFO [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C43999%2C1733691749022.meta:.meta(num 1733691783395) 2024-12-08T21:04:09,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741833_1009 (size=20031) 2024-12-08T21:04:09,664 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/WALs/229bab1f9d30,43999,1733691749022/229bab1f9d30%2C43999%2C1733691749022.1733691782933 not finished, retry = 0 2024-12-08T21:04:09,772 DEBUG [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/oldWALs 2024-12-08T21:04:09,772 INFO [RS:0;229bab1f9d30:43999 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 229bab1f9d30%2C43999%2C1733691749022:(num 1733691782933) 2024-12-08T21:04:09,772 DEBUG [RS:0;229bab1f9d30:43999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:04:09,773 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:04:09,773 INFO [RS:0;229bab1f9d30:43999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:04:09,774 INFO [RS:0;229bab1f9d30:43999 {}] hbase.ChoreService(370): Chore service for: regionserver/229bab1f9d30:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T21:04:09,774 INFO [RS:0;229bab1f9d30:43999 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:04:09,774 INFO [regionserver/229bab1f9d30:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:04:09,777 INFO [RS:0;229bab1f9d30:43999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43999 2024-12-08T21:04:09,783 INFO [regionserver/229bab1f9d30:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:04:09,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/229bab1f9d30,43999,1733691749022 2024-12-08T21:04:09,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:04:09,862 INFO [RS:0;229bab1f9d30:43999 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:04:09,874 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [229bab1f9d30,43999,1733691749022] 2024-12-08T21:04:09,886 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/229bab1f9d30,43999,1733691749022 already deleted, retry=false 2024-12-08T21:04:09,886 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 229bab1f9d30,43999,1733691749022 expired; onlineServers=0 2024-12-08T21:04:09,886 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '229bab1f9d30,41621,1733691748281' ***** 2024-12-08T21:04:09,886 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T21:04:09,886 INFO [M:0;229bab1f9d30:41621 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:04:09,887 INFO [M:0;229bab1f9d30:41621 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:04:09,887 DEBUG [M:0;229bab1f9d30:41621 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T21:04:09,888 DEBUG [M:0;229bab1f9d30:41621 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T21:04:09,888 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T21:04:09,888 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691752703 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.large.0-1733691752703,5,FailOnTimeoutGroup] 2024-12-08T21:04:09,888 DEBUG [master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691752708 {}] cleaner.HFileCleaner(306): Exit Thread[master/229bab1f9d30:0:becomeActiveMaster-HFileCleaner.small.0-1733691752708,5,FailOnTimeoutGroup] 2024-12-08T21:04:09,888 INFO [M:0;229bab1f9d30:41621 {}] hbase.ChoreService(370): Chore service for: master/229bab1f9d30:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T21:04:09,889 INFO [M:0;229bab1f9d30:41621 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:04:09,889 DEBUG [M:0;229bab1f9d30:41621 {}] master.HMaster(1795): Stopping service threads 2024-12-08T21:04:09,889 INFO [M:0;229bab1f9d30:41621 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T21:04:09,890 INFO [M:0;229bab1f9d30:41621 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:04:09,891 INFO [M:0;229bab1f9d30:41621 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T21:04:09,891 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T21:04:09,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T21:04:09,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:04:09,899 DEBUG [M:0;229bab1f9d30:41621 {}] zookeeper.ZKUtil(347): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T21:04:09,899 WARN [M:0;229bab1f9d30:41621 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T21:04:09,900 INFO [M:0;229bab1f9d30:41621 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/.lastflushedseqids 2024-12-08T21:04:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741868_1044 (size=712) 2024-12-08T21:04:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:04:09,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43999-0x100074a6af60001, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:04:09,976 INFO [RS:0;229bab1f9d30:43999 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:04:09,976 INFO [RS:0;229bab1f9d30:43999 {}] regionserver.HRegionServer(1031): Exiting; stopping=229bab1f9d30,43999,1733691749022; zookeeper connection closed. 2024-12-08T21:04:09,977 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@766ff8f1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@766ff8f1 2024-12-08T21:04:09,978 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T21:04:10,312 INFO [M:0;229bab1f9d30:41621 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T21:04:10,313 INFO [M:0;229bab1f9d30:41621 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T21:04:10,313 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:04:10,313 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:04:10,313 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:04:10,313 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:04:10,313 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:04:10,314 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.16 KB heapSize=191.20 KB 2024-12-08T21:04:10,345 DEBUG [M:0;229bab1f9d30:41621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d4d3975ba9cb4a86a48ac1b4aeaca6b5 is 82, key is hbase:meta,,1/info:regioninfo/1733691783477/Put/seqid=0 2024-12-08T21:04:10,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741869_1045 (size=5672) 2024-12-08T21:04:10,751 INFO [M:0;229bab1f9d30:41621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d4d3975ba9cb4a86a48ac1b4aeaca6b5 2024-12-08T21:04:10,781 DEBUG [M:0;229bab1f9d30:41621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/622ba8c101964cfc87368e425e5ffb2c is 813, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733691784779/Put/seqid=0 2024-12-08T21:04:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741870_1046 (size=14076) 2024-12-08T21:04:11,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:04:11,187 INFO [M:0;229bab1f9d30:41621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.60 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/622ba8c101964cfc87368e425e5ffb2c 2024-12-08T21:04:11,193 INFO [M:0;229bab1f9d30:41621 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 622ba8c101964cfc87368e425e5ffb2c 2024-12-08T21:04:11,207 DEBUG [M:0;229bab1f9d30:41621 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a57298f32774df5ae6e34fb44d8002f is 69, key is 229bab1f9d30,43999,1733691749022/rs:state/1733691752779/Put/seqid=0 2024-12-08T21:04:11,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741871_1047 (size=5156) 2024-12-08T21:04:11,614 INFO [M:0;229bab1f9d30:41621 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a57298f32774df5ae6e34fb44d8002f 2024-12-08T21:04:11,629 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d4d3975ba9cb4a86a48ac1b4aeaca6b5 as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d4d3975ba9cb4a86a48ac1b4aeaca6b5 2024-12-08T21:04:11,636 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d4d3975ba9cb4a86a48ac1b4aeaca6b5, entries=8, sequenceid=319, filesize=5.5 K 2024-12-08T21:04:11,638 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/622ba8c101964cfc87368e425e5ffb2c as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/622ba8c101964cfc87368e425e5ffb2c 2024-12-08T21:04:11,644 INFO [M:0;229bab1f9d30:41621 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 622ba8c101964cfc87368e425e5ffb2c 2024-12-08T21:04:11,644 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/622ba8c101964cfc87368e425e5ffb2c, entries=33, sequenceid=319, filesize=13.7 K 2024-12-08T21:04:11,645 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9a57298f32774df5ae6e34fb44d8002f as hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a57298f32774df5ae6e34fb44d8002f 2024-12-08T21:04:11,652 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42587/user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9a57298f32774df5ae6e34fb44d8002f, entries=1, sequenceid=319, filesize=5.0 K 2024-12-08T21:04:11,653 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.90 KB/195480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1340ms, sequenceid=319, compaction requested=false 2024-12-08T21:04:11,655 INFO [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:04:11,655 DEBUG [M:0;229bab1f9d30:41621 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733691850313Disabling compacts and flushes for region at 1733691850313Disabling writes for close at 1733691850313Obtaining lock to block concurrent updates at 1733691850314 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733691850314Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167075, getHeapSize=195720, getOffHeapSize=0, getCellsCount=366 at 1733691850315 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733691850331 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733691850331Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733691850345 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733691850345Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733691850758 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733691850780 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733691850780Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733691851193 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733691851207 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733691851207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@632031db: reopening flushed file at 1733691851628 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47740e9d: reopening flushed file at 1733691851636 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542f48ec: reopening flushed file at 1733691851644 (+8 ms)Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.90 KB/195480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1340ms, sequenceid=319, compaction requested=false at 1733691851653 (+9 ms)Writing region close event to WAL at 1733691851655 (+2 ms)Closed at 1733691851655 2024-12-08T21:04:11,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741830_1006 (size=188417) 2024-12-08T21:04:11,658 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7122e161-08fe-ad31-51e5-62edd498988b/MasterData/WALs/229bab1f9d30,41621,1733691748281/229bab1f9d30%2C41621%2C1733691748281.1733691751145 not finished, retry = 0 2024-12-08T21:04:11,759 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:04:11,759 INFO [M:0;229bab1f9d30:41621 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T21:04:11,760 INFO [M:0;229bab1f9d30:41621 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41621 2024-12-08T21:04:11,760 INFO [M:0;229bab1f9d30:41621 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:04:11,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:04:11,903 INFO [M:0;229bab1f9d30:41621 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:04:11,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41621-0x100074a6af60000, quorum=127.0.0.1:57515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:04:11,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T21:04:11,917 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T21:04:11,917 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T21:04:11,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T21:04:11,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.log.dir/,STOPPED} 2024-12-08T21:04:11,919 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T21:04:11,919 WARN [BP-170596891-172.17.0.2-1733691744660 heartbeating to localhost/127.0.0.1:42587 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T21:04:11,919 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T21:04:11,919 WARN [BP-170596891-172.17.0.2-1733691744660 heartbeating to localhost/127.0.0.1:42587 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-170596891-172.17.0.2-1733691744660 (Datanode Uuid dd967caa-a9b7-49bc-bc21-50988747a1e3) service to localhost/127.0.0.1:42587 2024-12-08T21:04:11,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/data/data1/current/BP-170596891-172.17.0.2-1733691744660 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T21:04:11,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/cluster_92a2855b-0600-d40c-8855-ecf80b32350f/data/data2/current/BP-170596891-172.17.0.2-1733691744660 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T21:04:11,921 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T21:04:11,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T21:04:11,929 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T21:04:11,929 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T21:04:11,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T21:04:11,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7cc54276-092f-6a04-0537-2da8f04ec121/hadoop.log.dir/,STOPPED} 2024-12-08T21:04:11,935 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T21:04:11,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down