2024-12-09 01:02:14,418 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 01:02:14,438 main DEBUG Took 0.017901 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 01:02:14,439 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 01:02:14,439 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 01:02:14,440 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 01:02:14,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,451 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 01:02:14,466 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,469 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,470 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,471 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,473 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,474 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,475 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,476 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,477 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:02:14,479 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,479 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 01:02:14,481 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:02:14,482 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 01:02:14,485 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 01:02:14,485 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 01:02:14,487 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 01:02:14,487 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 01:02:14,495 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 01:02:14,497 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 01:02:14,498 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 01:02:14,499 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 01:02:14,499 main DEBUG createAppenders(={Console}) 2024-12-09 01:02:14,500 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-09 01:02:14,500 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 01:02:14,500 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-09 01:02:14,501 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 01:02:14,501 main DEBUG OutputStream closed 2024-12-09 01:02:14,502 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 01:02:14,502 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 01:02:14,502 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-09 01:02:14,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 01:02:14,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 01:02:14,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 01:02:14,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 01:02:14,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 01:02:14,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 01:02:14,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 01:02:14,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 01:02:14,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 01:02:14,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 01:02:14,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 01:02:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 01:02:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 01:02:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 01:02:14,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 01:02:14,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 01:02:14,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 01:02:14,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 01:02:14,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 01:02:14,574 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-09 01:02:14,574 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 01:02:14,575 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-09T01:02:14,586 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-09 01:02:14,588 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 01:02:14,589 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T01:02:14,829 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5 2024-12-09T01:02:14,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T01:02:14,858 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f, deleteOnExit=true 2024-12-09T01:02:14,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T01:02:14,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/test.cache.data in system properties and HBase conf 2024-12-09T01:02:14,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T01:02:14,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.log.dir in system properties and HBase conf 2024-12-09T01:02:14,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T01:02:14,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T01:02:14,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T01:02:14,938 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T01:02:15,019 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:02:15,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T01:02:15,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T01:02:15,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T01:02:15,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T01:02:15,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T01:02:15,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T01:02:15,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T01:02:15,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T01:02:15,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T01:02:15,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/nfs.dump.dir in system properties and HBase conf 2024-12-09T01:02:15,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/java.io.tmpdir in system properties and HBase conf 2024-12-09T01:02:15,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T01:02:15,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T01:02:15,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T01:02:16,021 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T01:02:16,111 INFO [Time-limited test {}] log.Log(170): Logging initialized @2478ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T01:02:16,203 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:02:16,288 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T01:02:16,314 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T01:02:16,314 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T01:02:16,316 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T01:02:16,330 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:02:16,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.log.dir/,AVAILABLE} 2024-12-09T01:02:16,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T01:02:16,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@562f0d0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/java.io.tmpdir/jetty-localhost-46799-hadoop-hdfs-3_4_1-tests_jar-_-any-8770373898725150554/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T01:02:16,522 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:46799} 2024-12-09T01:02:16,522 INFO [Time-limited test {}] server.Server(415): Started @2890ms 2024-12-09T01:02:16,987 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:02:16,996 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T01:02:16,997 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T01:02:16,998 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T01:02:16,998 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T01:02:16,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.log.dir/,AVAILABLE} 2024-12-09T01:02:17,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T01:02:17,100 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc78d53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/java.io.tmpdir/jetty-localhost-40399-hadoop-hdfs-3_4_1-tests_jar-_-any-10436108667599882561/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T01:02:17,100 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:40399} 2024-12-09T01:02:17,100 INFO [Time-limited test {}] server.Server(415): Started @3468ms 2024-12-09T01:02:17,149 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T01:02:18,004 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/data/data2/current/BP-1050418544-172.17.0.2-1733706135545/current, will proceed with Du for space computation calculation, 2024-12-09T01:02:18,004 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/data/data1/current/BP-1050418544-172.17.0.2-1733706135545/current, will proceed with Du for space computation calculation, 2024-12-09T01:02:18,028 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T01:02:18,071 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5ae32092a6edcd71 with lease ID 0xf78b28eaaff9f421: Processing first storage report for DS-3489e4cd-3466-438d-aaff-34362cbee96a from datanode DatanodeRegistration(127.0.0.1:44185, datanodeUuid=096d782a-4d07-48af-9b90-b523295ed21d, infoPort=37353, infoSecurePort=0, ipcPort=42743, storageInfo=lv=-57;cid=testClusterID;nsid=7655606;c=1733706135545) 2024-12-09T01:02:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ae32092a6edcd71 with lease ID 0xf78b28eaaff9f421: from storage DS-3489e4cd-3466-438d-aaff-34362cbee96a node DatanodeRegistration(127.0.0.1:44185, datanodeUuid=096d782a-4d07-48af-9b90-b523295ed21d, infoPort=37353, infoSecurePort=0, ipcPort=42743, storageInfo=lv=-57;cid=testClusterID;nsid=7655606;c=1733706135545), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T01:02:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5ae32092a6edcd71 with lease ID 0xf78b28eaaff9f421: Processing first storage report for DS-d36c2d77-afee-4c9a-8ca3-e69b14b5e8b5 from datanode DatanodeRegistration(127.0.0.1:44185, datanodeUuid=096d782a-4d07-48af-9b90-b523295ed21d, infoPort=37353, infoSecurePort=0, ipcPort=42743, storageInfo=lv=-57;cid=testClusterID;nsid=7655606;c=1733706135545) 2024-12-09T01:02:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ae32092a6edcd71 with lease ID 0xf78b28eaaff9f421: from storage DS-d36c2d77-afee-4c9a-8ca3-e69b14b5e8b5 node DatanodeRegistration(127.0.0.1:44185, datanodeUuid=096d782a-4d07-48af-9b90-b523295ed21d, infoPort=37353, infoSecurePort=0, ipcPort=42743, storageInfo=lv=-57;cid=testClusterID;nsid=7655606;c=1733706135545), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T01:02:18,125 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5 2024-12-09T01:02:18,192 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/zookeeper_0, clientPort=57232, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T01:02:18,201 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57232 2024-12-09T01:02:18,210 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:18,212 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:18,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741825_1001 (size=7) 2024-12-09T01:02:18,806 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 with version=8 2024-12-09T01:02:18,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/hbase-staging 2024-12-09T01:02:18,905 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T01:02:19,215 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1bae0942de96:0 server-side Connection retries=9 2024-12-09T01:02:19,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:19,228 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:19,234 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:02:19,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:19,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:02:19,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T01:02:19,460 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T01:02:19,468 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T01:02:19,471 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:02:19,492 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 112775 (auto-detected) 2024-12-09T01:02:19,493 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T01:02:19,508 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34071 2024-12-09T01:02:19,533 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34071 connecting to ZooKeeper ensemble=127.0.0.1:57232 2024-12-09T01:02:19,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340710x0, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:02:19,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34071-0x100082600dc0000 connected 2024-12-09T01:02:19,977 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:19,979 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:19,991 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:02:19,996 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0, hbase.cluster.distributed=false 2024-12-09T01:02:20,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:02:20,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34071 2024-12-09T01:02:20,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34071 2024-12-09T01:02:20,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34071 2024-12-09T01:02:20,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34071 2024-12-09T01:02:20,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34071 2024-12-09T01:02:20,166 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1bae0942de96:0 server-side Connection retries=9 2024-12-09T01:02:20,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:20,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:20,168 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:02:20,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:02:20,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:02:20,170 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:02:20,173 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:02:20,174 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46191 2024-12-09T01:02:20,176 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46191 connecting to ZooKeeper ensemble=127.0.0.1:57232 2024-12-09T01:02:20,177 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:20,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:20,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461910x0, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:02:20,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461910x0, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:02:20,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46191-0x100082600dc0001 connected 2024-12-09T01:02:20,205 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:02:20,213 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:02:20,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:02:20,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:02:20,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-09T01:02:20,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46191 2024-12-09T01:02:20,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46191 2024-12-09T01:02:20,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-09T01:02:20,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-09T01:02:20,245 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1bae0942de96:34071 2024-12-09T01:02:20,246 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1bae0942de96,34071,1733706138987 2024-12-09T01:02:20,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:02:20,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:02:20,278 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1bae0942de96,34071,1733706138987 2024-12-09T01:02:20,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:02:20,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:20,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:20,309 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T01:02:20,311 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1bae0942de96,34071,1733706138987 from backup master directory 2024-12-09T01:02:20,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1bae0942de96,34071,1733706138987 2024-12-09T01:02:20,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:02:20,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:02:20,325 WARN [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:02:20,326 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1bae0942de96,34071,1733706138987 2024-12-09T01:02:20,329 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T01:02:20,330 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T01:02:20,381 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/hbase.id] with ID: e806d327-383e-4d31-adbb-8990f87ca90d 2024-12-09T01:02:20,381 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/.tmp/hbase.id 2024-12-09T01:02:20,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741826_1002 (size=42) 2024-12-09T01:02:20,791 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/.tmp/hbase.id]:[hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/hbase.id] 2024-12-09T01:02:20,842 INFO [master/1bae0942de96:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:02:20,848 INFO [master/1bae0942de96:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T01:02:20,867 INFO [master/1bae0942de96:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-09T01:02:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:20,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741827_1003 (size=196) 2024-12-09T01:02:20,917 INFO [master/1bae0942de96:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:02:20,922 INFO [master/1bae0942de96:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T01:02:20,937 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:02:20,941 INFO [master/1bae0942de96:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:02:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741828_1004 (size=1189) 2024-12-09T01:02:21,389 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store 2024-12-09T01:02:21,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741829_1005 (size=34) 2024-12-09T01:02:21,827 INFO [master/1bae0942de96:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T01:02:21,830 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:21,831 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:02:21,831 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:02:21,831 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:02:21,832 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:02:21,832 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:02:21,832 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:02:21,833 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733706141830Disabling compacts and flushes for region at 1733706141830Disabling writes for close at 1733706141832 (+2 ms)Writing region close event to WAL at 1733706141832Closed at 1733706141832 2024-12-09T01:02:21,835 WARN [master/1bae0942de96:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/.initializing 2024-12-09T01:02:21,835 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/WALs/1bae0942de96,34071,1733706138987 2024-12-09T01:02:21,843 INFO [master/1bae0942de96:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:02:21,857 INFO [master/1bae0942de96:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1bae0942de96%2C34071%2C1733706138987, suffix=, logDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/WALs/1bae0942de96,34071,1733706138987, archiveDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/oldWALs, maxLogs=10 2024-12-09T01:02:21,877 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/WALs/1bae0942de96,34071,1733706138987/1bae0942de96%2C34071%2C1733706138987.1733706141861, exclude list is [], retry=0 2024-12-09T01:02:21,891 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44185,DS-3489e4cd-3466-438d-aaff-34362cbee96a,DISK] 2024-12-09T01:02:21,894 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T01:02:21,923 INFO [master/1bae0942de96:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/WALs/1bae0942de96,34071,1733706138987/1bae0942de96%2C34071%2C1733706138987.1733706141861 2024-12-09T01:02:21,924 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37353:37353)] 2024-12-09T01:02:21,925 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:02:21,925 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:21,928 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:21,928 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:21,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:21,982 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T01:02:21,986 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:21,988 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:21,989 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:21,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T01:02:21,992 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:21,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:02:21,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:21,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T01:02:21,997 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:21,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:02:21,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T01:02:22,001 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:22,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:02:22,002 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,006 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,007 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,013 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,013 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,017 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:02:22,021 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:02:22,026 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:02:22,027 INFO [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65137894, jitterRate=-0.029369741678237915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:02:22,033 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733706141939Initializing all the Stores at 1733706141941 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706141942 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706141942Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706141943 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706141943Cleaning up temporary data from old regions at 1733706142013 (+70 ms)Region opened successfully at 1733706142033 (+20 ms) 2024-12-09T01:02:22,034 INFO [master/1bae0942de96:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T01:02:22,061 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762a4217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1bae0942de96/172.17.0.2:0 2024-12-09T01:02:22,087 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T01:02:22,096 INFO [master/1bae0942de96:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T01:02:22,096 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T01:02:22,098 INFO [master/1bae0942de96:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T01:02:22,100 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T01:02:22,104 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T01:02:22,105 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T01:02:22,131 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T01:02:22,139 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T01:02:22,182 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T01:02:22,185 INFO [master/1bae0942de96:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T01:02:22,187 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T01:02:22,199 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T01:02:22,202 INFO [master/1bae0942de96:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T01:02:22,207 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T01:02:22,215 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T01:02:22,217 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T01:02:22,224 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T01:02:22,243 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T01:02:22,248 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T01:02:22,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:02:22,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:02:22,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,260 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1bae0942de96,34071,1733706138987, sessionid=0x100082600dc0000, setting cluster-up flag (Was=false) 2024-12-09T01:02:22,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,316 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T01:02:22,319 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1bae0942de96,34071,1733706138987 2024-12-09T01:02:22,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:22,366 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T01:02:22,370 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1bae0942de96,34071,1733706138987 2024-12-09T01:02:22,377 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T01:02:22,435 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(746): ClusterId : e806d327-383e-4d31-adbb-8990f87ca90d 2024-12-09T01:02:22,438 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:02:22,439 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T01:02:22,448 INFO [master/1bae0942de96:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T01:02:22,453 INFO [master/1bae0942de96:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T01:02:22,459 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:02:22,459 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:02:22,458 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1bae0942de96,34071,1733706138987 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T01:02:22,467 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:02:22,467 DEBUG [RS:0;1bae0942de96:46191 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431c6eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1bae0942de96/172.17.0.2:0 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1bae0942de96:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1bae0942de96:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1bae0942de96:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1bae0942de96:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1bae0942de96:0, corePoolSize=10, maxPoolSize=10 2024-12-09T01:02:22,468 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,469 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1bae0942de96:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:02:22,469 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,474 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733706172474 2024-12-09T01:02:22,475 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:02:22,475 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T01:02:22,476 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T01:02:22,476 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T01:02:22,480 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T01:02:22,481 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T01:02:22,481 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T01:02:22,481 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T01:02:22,483 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:22,483 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T01:02:22,482 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,485 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1bae0942de96:46191 2024-12-09T01:02:22,486 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T01:02:22,487 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T01:02:22,488 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T01:02:22,489 INFO [RS:0;1bae0942de96:46191 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:02:22,489 INFO [RS:0;1bae0942de96:46191 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:02:22,490 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T01:02:22,490 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:02:22,490 INFO [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T01:02:22,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741831_1007 (size=1321) 2024-12-09T01:02:22,493 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(2659): reportForDuty to master=1bae0942de96,34071,1733706138987 with port=46191, startcode=1733706140132 2024-12-09T01:02:22,493 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.large.0-1733706142491,5,FailOnTimeoutGroup] 2024-12-09T01:02:22,494 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.small.0-1733706142493,5,FailOnTimeoutGroup] 2024-12-09T01:02:22,494 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,494 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T01:02:22,495 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,495 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,502 DEBUG [RS:0;1bae0942de96:46191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:02:22,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:02:22,535 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34071 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,537 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34071 {}] master.ServerManager(517): Registering regionserver=1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,550 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:02:22,550 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41839 2024-12-09T01:02:22,550 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:02:22,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:02:22,583 DEBUG [RS:0;1bae0942de96:46191 {}] zookeeper.ZKUtil(111): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,583 WARN [RS:0;1bae0942de96:46191 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:02:22,583 INFO [RS:0;1bae0942de96:46191 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:02:22,584 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,586 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1bae0942de96,46191,1733706140132] 2024-12-09T01:02:22,608 INFO [RS:0;1bae0942de96:46191 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:02:22,620 INFO [RS:0;1bae0942de96:46191 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:02:22,624 INFO [RS:0;1bae0942de96:46191 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:02:22,624 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,625 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:02:22,630 INFO [RS:0;1bae0942de96:46191 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:02:22,631 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,631 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,631 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,631 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1bae0942de96:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,632 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,633 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1bae0942de96:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:02:22,633 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1bae0942de96:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:02:22,633 DEBUG [RS:0;1bae0942de96:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1bae0942de96:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:02:22,634 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,634 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,635 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,635 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,635 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,635 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,46191,1733706140132-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:02:22,652 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:02:22,654 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,46191,1733706140132-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,654 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,654 INFO [RS:0;1bae0942de96:46191 {}] regionserver.Replication(171): 1bae0942de96,46191,1733706140132 started 2024-12-09T01:02:22,671 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:22,671 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1482): Serving as 1bae0942de96,46191,1733706140132, RpcServer on 1bae0942de96/172.17.0.2:46191, sessionid=0x100082600dc0001 2024-12-09T01:02:22,672 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:02:22,672 DEBUG [RS:0;1bae0942de96:46191 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,672 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1bae0942de96,46191,1733706140132' 2024-12-09T01:02:22,673 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:02:22,674 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:02:22,674 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:02:22,674 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:02:22,675 DEBUG [RS:0;1bae0942de96:46191 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1bae0942de96,46191,1733706140132 2024-12-09T01:02:22,675 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1bae0942de96,46191,1733706140132' 2024-12-09T01:02:22,675 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:02:22,675 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:02:22,676 DEBUG [RS:0;1bae0942de96:46191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:02:22,676 INFO [RS:0;1bae0942de96:46191 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:02:22,676 INFO [RS:0;1bae0942de96:46191 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:02:22,897 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T01:02:22,899 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:02:22,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741832_1008 (size=32) 2024-12-09T01:02:23,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:23,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T01:02:23,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:02:23,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:23,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:23,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T01:02:23,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:02:23,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:23,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:23,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T01:02:23,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:02:23,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:23,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:23,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T01:02:23,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:02:23,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:23,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:23,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:02:23,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740 2024-12-09T01:02:23,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740 2024-12-09T01:02:23,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:02:23,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:02:23,351 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:02:23,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:02:23,357 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:02:23,358 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71400730, jitterRate=0.06395378708839417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:02:23,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733706143321Initializing all the Stores at 1733706143326 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706143326Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706143327 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706143327Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706143327Cleaning up temporary data from old regions at 1733706143350 (+23 ms)Region opened successfully at 1733706143360 (+10 ms) 2024-12-09T01:02:23,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:02:23,360 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:02:23,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:02:23,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:02:23,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:02:23,362 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:02:23,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733706143360Disabling compacts and flushes for region at 1733706143360Disabling writes for close at 1733706143361 (+1 ms)Writing region close event to WAL at 1733706143362 (+1 ms)Closed at 1733706143362 2024-12-09T01:02:23,365 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:02:23,365 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T01:02:23,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T01:02:23,377 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:02:23,380 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T01:02:23,532 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:24,533 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:25,534 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:26,535 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:27,536 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:28,537 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:29,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T01:02:29,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T01:02:29,538 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:30,539 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:31,540 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:32,540 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:33,541 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:34,542 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:35,543 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:36,544 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:37,545 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:38,545 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:39,546 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:40,547 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:41,547 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:42,548 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:43,549 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:44,550 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:45,551 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:46,552 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:47,552 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:48,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:02:48,553 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:49,554 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:50,555 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:51,556 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:52,557 WARN [1bae0942de96:34071 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:02:52,687 INFO [RS:0;1bae0942de96:46191 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:02:52,691 INFO [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1bae0942de96%2C46191%2C1733706140132, suffix=, logDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132, archiveDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/oldWALs, maxLogs=32 2024-12-09T01:02:52,705 DEBUG [RS:0;1bae0942de96:46191 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132/1bae0942de96%2C46191%2C1733706140132.1733706172693, exclude list is [], retry=0 2024-12-09T01:02:52,709 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44185,DS-3489e4cd-3466-438d-aaff-34362cbee96a,DISK] 2024-12-09T01:02:52,712 INFO [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132/1bae0942de96%2C46191%2C1733706140132.1733706172693 2024-12-09T01:02:52,713 DEBUG [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37353:37353)] 2024-12-09T01:02:52,812 DEBUG [1bae0942de96:34071 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T01:02:52,824 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:52,829 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1bae0942de96,46191,1733706140132, state=OPENING 2024-12-09T01:02:52,902 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T01:02:52,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:52,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:02:52,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:02:52,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:02:52,915 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:02:52,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:02:53,098 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:02:53,100 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36893, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:02:53,110 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T01:02:53,111 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:02:53,112 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T01:02:53,115 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1bae0942de96%2C46191%2C1733706140132.meta, suffix=.meta, logDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132, archiveDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/oldWALs, maxLogs=32 2024-12-09T01:02:53,132 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132/1bae0942de96%2C46191%2C1733706140132.meta.1733706173117.meta, exclude list is [], retry=0 2024-12-09T01:02:53,136 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44185,DS-3489e4cd-3466-438d-aaff-34362cbee96a,DISK] 2024-12-09T01:02:53,139 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/WALs/1bae0942de96,46191,1733706140132/1bae0942de96%2C46191%2C1733706140132.meta.1733706173117.meta 2024-12-09T01:02:53,139 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37353:37353)] 2024-12-09T01:02:53,140 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:02:53,141 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T01:02:53,143 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T01:02:53,147 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T01:02:53,151 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T01:02:53,151 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:53,151 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T01:02:53,152 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T01:02:53,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T01:02:53,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:02:53,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:53,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:53,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T01:02:53,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:02:53,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:53,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:53,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T01:02:53,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:02:53,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:53,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:53,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T01:02:53,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:02:53,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:53,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:02:53,164 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:02:53,166 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740 2024-12-09T01:02:53,169 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740 2024-12-09T01:02:53,171 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:02:53,171 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:02:53,172 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:02:53,174 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:02:53,176 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59464653, jitterRate=-0.11390762031078339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:02:53,176 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T01:02:53,177 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733706173152Writing region info on filesystem at 1733706173152Initializing all the Stores at 1733706173154 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706173154Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706173154Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706173154Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733706173154Cleaning up temporary data from old regions at 1733706173171 (+17 ms)Running coprocessor post-open hooks at 1733706173176 (+5 ms)Region opened successfully at 1733706173177 (+1 ms) 2024-12-09T01:02:53,183 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733706173089 2024-12-09T01:02:53,194 DEBUG [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T01:02:53,194 INFO [RS_OPEN_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T01:02:53,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:53,198 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1bae0942de96,46191,1733706140132, state=OPEN 2024-12-09T01:02:53,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:02:53,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:02:53,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:02:53,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:02:53,248 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1bae0942de96,46191,1733706140132 2024-12-09T01:02:53,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T01:02:53,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1bae0942de96,46191,1733706140132 in 330 msec 2024-12-09T01:02:53,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T01:02:53,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8850 sec 2024-12-09T01:02:53,263 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:02:53,263 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T01:02:53,279 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:02:53,280 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1bae0942de96,46191,1733706140132, seqNum=-1] 2024-12-09T01:02:53,296 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:02:53,299 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:02:53,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9110 sec 2024-12-09T01:02:53,317 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733706173317, completionTime=-1 2024-12-09T01:02:53,319 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T01:02:53,319 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T01:02:53,343 INFO [master/1bae0942de96:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T01:02:53,343 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733706233343 2024-12-09T01:02:53,343 INFO [master/1bae0942de96:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733706293343 2024-12-09T01:02:53,343 INFO [master/1bae0942de96:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-09T01:02:53,345 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,345 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,345 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,346 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1bae0942de96:34071, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,346 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,347 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,353 DEBUG [master/1bae0942de96:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T01:02:53,375 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.049sec 2024-12-09T01:02:53,377 INFO [master/1bae0942de96:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T01:02:53,378 INFO [master/1bae0942de96:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T01:02:53,379 INFO [master/1bae0942de96:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T01:02:53,380 INFO [master/1bae0942de96:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T01:02:53,380 INFO [master/1bae0942de96:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T01:02:53,381 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:02:53,382 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T01:02:53,389 DEBUG [master/1bae0942de96:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T01:02:53,389 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T01:02:53,390 INFO [master/1bae0942de96:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1bae0942de96,34071,1733706138987-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:02:53,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e818d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,484 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T01:02:53,484 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T01:02:53,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1bae0942de96,34071,-1 for getting cluster id 2024-12-09T01:02:53,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:02:53,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e806d327-383e-4d31-adbb-8990f87ca90d' 2024-12-09T01:02:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:02:53,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e806d327-383e-4d31-adbb-8990f87ca90d" 2024-12-09T01:02:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34742aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1bae0942de96,34071,-1] 2024-12-09T01:02:53,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:02:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:02:53,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:02:53,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562ba9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:02:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1bae0942de96,46191,1733706140132, seqNum=-1] 2024-12-09T01:02:53,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:02:53,519 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:02:53,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1bae0942de96,34071,1733706138987 2024-12-09T01:02:53,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14688263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1bae0942de96,34071,-1 for getting cluster id 2024-12-09T01:02:53,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:02:53,542 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e806d327-383e-4d31-adbb-8990f87ca90d' 2024-12-09T01:02:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:02:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e806d327-383e-4d31-adbb-8990f87ca90d" 2024-12-09T01:02:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a2feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1bae0942de96,34071,-1] 2024-12-09T01:02:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:02:53,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:02:53,546 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:02:53,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e44f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:02:53,560 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=16430 2024-12-09T01:02:53,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:02:53,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1bae0942de96,34071,1733706138987 2024-12-09T01:02:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55a08759 2024-12-09T01:02:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:02:53,577 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:02:53,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:02:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T01:02:53,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:02:53,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-09T01:02:53,594 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:53,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:02:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:02:53,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741835_1011 (size=428) 2024-12-09T01:02:54,016 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17a3f7298b74b7c3bf26405f5118df93, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:02:54,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741836_1012 (size=79) 2024-12-09T01:02:54,429 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:54,430 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 17a3f7298b74b7c3bf26405f5118df93, disabling compactions & flushes 2024-12-09T01:02:54,431 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,431 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,431 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. after waiting 0 ms 2024-12-09T01:02:54,431 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,432 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,432 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17a3f7298b74b7c3bf26405f5118df93: Waiting for close lock at 1733706174430Disabling compacts and flushes for region at 1733706174430Disabling writes for close at 1733706174431 (+1 ms)Writing region close event to WAL at 1733706174431Closed at 1733706174432 (+1 ms) 2024-12-09T01:02:54,436 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:02:54,440 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733706174436"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706174436"}]},"ts":"1733706174436"} 2024-12-09T01:02:54,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:02:54,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:02:54,449 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706174447"}]},"ts":"1733706174447"} 2024-12-09T01:02:54,453 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-09T01:02:54,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=17a3f7298b74b7c3bf26405f5118df93, ASSIGN}] 2024-12-09T01:02:54,458 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=17a3f7298b74b7c3bf26405f5118df93, ASSIGN 2024-12-09T01:02:54,459 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=17a3f7298b74b7c3bf26405f5118df93, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:02:54,612 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=17a3f7298b74b7c3bf26405f5118df93, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:02:54,618 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=17a3f7298b74b7c3bf26405f5118df93, ASSIGN because future has completed 2024-12-09T01:02:54,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a3f7298b74b7c3bf26405f5118df93, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:02:54,785 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,785 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 17a3f7298b74b7c3bf26405f5118df93, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:02:54,786 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,786 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:54,786 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,786 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,789 INFO [StoreOpener-17a3f7298b74b7c3bf26405f5118df93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,807 INFO [StoreOpener-17a3f7298b74b7c3bf26405f5118df93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17a3f7298b74b7c3bf26405f5118df93 columnFamilyName testFamily 2024-12-09T01:02:54,807 DEBUG [StoreOpener-17a3f7298b74b7c3bf26405f5118df93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:54,809 INFO [StoreOpener-17a3f7298b74b7c3bf26405f5118df93-1 {}] regionserver.HStore(327): Store=17a3f7298b74b7c3bf26405f5118df93/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:02:54,809 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,811 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,812 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,813 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,813 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,816 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,820 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:02:54,821 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 17a3f7298b74b7c3bf26405f5118df93; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69557717, jitterRate=0.036490753293037415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:02:54,821 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:02:54,823 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 17a3f7298b74b7c3bf26405f5118df93: Running coprocessor pre-open hook at 1733706174786Writing region info on filesystem at 1733706174786Initializing all the Stores at 1733706174788 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706174788Cleaning up temporary data from old regions at 1733706174813 (+25 ms)Running coprocessor post-open hooks at 1733706174822 (+9 ms)Region opened successfully at 1733706174823 (+1 ms) 2024-12-09T01:02:54,825 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93., pid=6, masterSystemTime=1733706174774 2024-12-09T01:02:54,828 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,829 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:02:54,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=17a3f7298b74b7c3bf26405f5118df93, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:54,836 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a3f7298b74b7c3bf26405f5118df93, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:02:54,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T01:02:54,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 17a3f7298b74b7c3bf26405f5118df93, server=1bae0942de96,46191,1733706140132 in 219 msec 2024-12-09T01:02:54,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T01:02:54,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=17a3f7298b74b7c3bf26405f5118df93, ASSIGN in 386 msec 2024-12-09T01:02:54,846 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:02:54,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706174846"}]},"ts":"1733706174846"} 2024-12-09T01:02:54,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-09T01:02:54,851 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:02:54,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2660 sec 2024-12-09T01:02:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:02:56,648 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-09T01:02:56,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-09T01:02:56,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:02:56,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-09T01:02:56,657 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:02:56,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-09T01:02:56,661 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:02:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1bae0942de96,46191,1733706140132, seqNum=-1] 2024-12-09T01:02:56,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:02:56,666 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:02:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:02:56,686 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:02:56,689 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-09T01:02:56,693 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T01:02:56,697 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T01:02:56,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T01:02:56,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=161894076348629000 2024-12-09T01:02:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706179700, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:02:56,724 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T01:02:56,725 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T01:02:56,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:02:56,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:02:56,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=161894076348629002 2024-12-09T01:02:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706179727, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:02:56,732 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:02:56,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=161894076348629003 2024-12-09T01:02:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706179732, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:02:56,736 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:02:56,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=161894076348629004 2024-12-09T01:02:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706179736, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:02:56,741 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:02:56,750 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1bae0942de96:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-106127452_22 at /127.0.0.1:36882 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=16381 (was 16430) 2024-12-09T01:02:56,758 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=16378 2024-12-09T01:02:56,761 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:02:56,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-09T01:02:56,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:02:56,766 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:56,766 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-09T01:02:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:02:56,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:02:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741837_1013 (size=414) 2024-12-09T01:02:57,187 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aba63087cedfb13adaecd45bf828b8ff, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:02:57,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741838_1014 (size=65) 2024-12-09T01:02:57,602 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:57,602 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing aba63087cedfb13adaecd45bf828b8ff, disabling compactions & flushes 2024-12-09T01:02:57,602 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,602 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,603 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. after waiting 1 ms 2024-12-09T01:02:57,603 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,603 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,603 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for aba63087cedfb13adaecd45bf828b8ff: Waiting for close lock at 1733706177602Disabling compacts and flushes for region at 1733706177602Disabling writes for close at 1733706177603 (+1 ms)Writing region close event to WAL at 1733706177603Closed at 1733706177603 2024-12-09T01:02:57,606 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:02:57,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733706177607"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706177607"}]},"ts":"1733706177607"} 2024-12-09T01:02:57,612 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:02:57,614 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:02:57,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706177614"}]},"ts":"1733706177614"} 2024-12-09T01:02:57,618 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T01:02:57,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=aba63087cedfb13adaecd45bf828b8ff, ASSIGN}] 2024-12-09T01:02:57,620 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=aba63087cedfb13adaecd45bf828b8ff, ASSIGN 2024-12-09T01:02:57,621 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=aba63087cedfb13adaecd45bf828b8ff, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:02:57,773 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=aba63087cedfb13adaecd45bf828b8ff, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:57,782 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=aba63087cedfb13adaecd45bf828b8ff, ASSIGN because future has completed 2024-12-09T01:02:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:02:57,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba63087cedfb13adaecd45bf828b8ff, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:02:57,947 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,947 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => aba63087cedfb13adaecd45bf828b8ff, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:02:57,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:02:57,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,952 INFO [StoreOpener-aba63087cedfb13adaecd45bf828b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,955 INFO [StoreOpener-aba63087cedfb13adaecd45bf828b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aba63087cedfb13adaecd45bf828b8ff columnFamilyName testFamily 2024-12-09T01:02:57,955 DEBUG [StoreOpener-aba63087cedfb13adaecd45bf828b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:02:57,956 INFO [StoreOpener-aba63087cedfb13adaecd45bf828b8ff-1 {}] regionserver.HStore(327): Store=aba63087cedfb13adaecd45bf828b8ff/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:02:57,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,958 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,958 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,962 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,965 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:02:57,965 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened aba63087cedfb13adaecd45bf828b8ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75197692, jitterRate=0.12053292989730835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:02:57,966 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:02:57,967 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for aba63087cedfb13adaecd45bf828b8ff: Running coprocessor pre-open hook at 1733706177949Writing region info on filesystem at 1733706177949Initializing all the Stores at 1733706177951 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706177951Cleaning up temporary data from old regions at 1733706177959 (+8 ms)Running coprocessor post-open hooks at 1733706177966 (+7 ms)Region opened successfully at 1733706177966 2024-12-09T01:02:57,968 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., pid=9, masterSystemTime=1733706177939 2024-12-09T01:02:57,971 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,971 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:02:57,972 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=aba63087cedfb13adaecd45bf828b8ff, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:02:57,976 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure aba63087cedfb13adaecd45bf828b8ff, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:02:57,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-09T01:02:57,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure aba63087cedfb13adaecd45bf828b8ff, server=1bae0942de96,46191,1733706140132 in 193 msec 2024-12-09T01:02:57,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T01:02:57,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=aba63087cedfb13adaecd45bf828b8ff, ASSIGN in 362 msec 2024-12-09T01:02:57,985 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:02:57,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706177985"}]},"ts":"1733706177985"} 2024-12-09T01:02:57,988 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T01:02:57,990 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:02:57,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2290 sec 2024-12-09T01:02:58,687 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:02:58,710 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-09T01:02:58,711 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T01:02:58,712 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-09T01:02:59,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T01:02:59,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T01:02:59,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:02:59,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T01:02:59,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T01:02:59,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T01:02:59,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-09T01:02:59,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T01:02:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:02:59,796 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-09T01:02:59,796 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T01:02:59,797 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:02:59,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T01:02:59,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:02:59,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-09T01:02:59,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:02:59,821 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:02:59,821 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T01:02:59,825 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T01:02:59,826 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T01:03:05,847 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6016ms, rpcTimeout=5999ms 2024-12-09T01:03:05,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6016ms, rpcTimeout=5999ms 2024-12-09T01:03:05,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:03:05,856 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=381 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=16367 (was 16378) 2024-12-09T01:03:05,863 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=381, ProcessCount=11, AvailableMemoryMB=16366 2024-12-09T01:03:05,866 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:05,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-09T01:03:05,870 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:05,870 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:05,871 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-09T01:03:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T01:03:05,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:05,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741839_1015 (size=412) 2024-12-09T01:03:06,284 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68c0546b8ae3c545b7fd61a2ea54459d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:06,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741840_1016 (size=63) 2024-12-09T01:03:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-09T01:03:06,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:06,698 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 68c0546b8ae3c545b7fd61a2ea54459d, disabling compactions & flushes 2024-12-09T01:03:06,698 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:06,698 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:06,698 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. after waiting 0 ms 2024-12-09T01:03:06,698 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:06,698 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:06,698 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68c0546b8ae3c545b7fd61a2ea54459d: Waiting for close lock at 1733706186698Disabling compacts and flushes for region at 1733706186698Disabling writes for close at 1733706186698Writing region close event to WAL at 1733706186698Closed at 1733706186698 2024-12-09T01:03:06,701 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:06,702 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733706186702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706186702"}]},"ts":"1733706186702"} 2024-12-09T01:03:06,708 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:06,710 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:06,710 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706186710"}]},"ts":"1733706186710"} 2024-12-09T01:03:06,714 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T01:03:06,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=68c0546b8ae3c545b7fd61a2ea54459d, ASSIGN}] 2024-12-09T01:03:06,717 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=68c0546b8ae3c545b7fd61a2ea54459d, ASSIGN 2024-12-09T01:03:06,719 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=68c0546b8ae3c545b7fd61a2ea54459d, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:06,870 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=68c0546b8ae3c545b7fd61a2ea54459d, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:06,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=68c0546b8ae3c545b7fd61a2ea54459d, ASSIGN because future has completed 2024-12-09T01:03:06,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68c0546b8ae3c545b7fd61a2ea54459d, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:06,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T01:03:07,040 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:07,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 68c0546b8ae3c545b7fd61a2ea54459d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:07,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:07,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,041 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,044 INFO [StoreOpener-68c0546b8ae3c545b7fd61a2ea54459d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,046 INFO [StoreOpener-68c0546b8ae3c545b7fd61a2ea54459d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68c0546b8ae3c545b7fd61a2ea54459d columnFamilyName testFamily 2024-12-09T01:03:07,046 DEBUG [StoreOpener-68c0546b8ae3c545b7fd61a2ea54459d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:07,047 INFO [StoreOpener-68c0546b8ae3c545b7fd61a2ea54459d-1 {}] regionserver.HStore(327): Store=68c0546b8ae3c545b7fd61a2ea54459d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:07,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,050 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,051 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,051 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,055 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:07,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 68c0546b8ae3c545b7fd61a2ea54459d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73441229, jitterRate=0.09435959160327911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:07,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:03:07,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 68c0546b8ae3c545b7fd61a2ea54459d: Running coprocessor pre-open hook at 1733706187041Writing region info on filesystem at 1733706187041Initializing all the Stores at 1733706187043 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706187043Cleaning up temporary data from old regions at 1733706187051 (+8 ms)Running coprocessor post-open hooks at 1733706187061 (+10 ms)Region opened successfully at 1733706187061 2024-12-09T01:03:07,063 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., pid=12, masterSystemTime=1733706187035 2024-12-09T01:03:07,066 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:07,067 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:03:07,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=68c0546b8ae3c545b7fd61a2ea54459d, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:07,071 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68c0546b8ae3c545b7fd61a2ea54459d, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:07,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T01:03:07,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 68c0546b8ae3c545b7fd61a2ea54459d, server=1bae0942de96,46191,1733706140132 in 193 msec 2024-12-09T01:03:07,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T01:03:07,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=68c0546b8ae3c545b7fd61a2ea54459d, ASSIGN in 362 msec 2024-12-09T01:03:07,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:07,081 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706187081"}]},"ts":"1733706187081"} 2024-12-09T01:03:07,083 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T01:03:07,085 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:07,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2190 sec 2024-12-09T01:03:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T01:03:08,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-09T01:03:08,896 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T01:03:08,897 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:08,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T01:03:08,903 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:08,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-09T01:03:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:08,919 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:08,919 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T01:03:08,922 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T01:03:08,925 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T01:03:08,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T01:03:08,927 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T01:03:09,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-09T01:03:09,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:10,714 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-09T01:03:12,566 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:14,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629013' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T01:03:14,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43206 deadline: 1733706200939, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629013' does not exist 2024-12-09T01:03:14,944 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1bae0942de96,46191,1733706140132 for closing scanner id = 161894076348629013 for 68c0546b8ae3c545b7fd61a2ea54459d of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629013' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:03:14,947 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=322 (was 381), ProcessCount=11 (was 11), AvailableMemoryMB=16350 (was 16366) 2024-12-09T01:03:14,955 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=322, ProcessCount=11, AvailableMemoryMB=16350 2024-12-09T01:03:14,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-09T01:03:14,962 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:14,962 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:14,963 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-09T01:03:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:03:14,964 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:14,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741841_1017 (size=423) 2024-12-09T01:03:15,381 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5fdfc6cd0e1d087d33da2039f5cfb09e, NAME => 'testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:15,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-09T01:03:15,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741842_1018 (size=74) 2024-12-09T01:03:15,838 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:15,839 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 5fdfc6cd0e1d087d33da2039f5cfb09e, disabling compactions & flushes 2024-12-09T01:03:15,839 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:15,839 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:15,840 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. after waiting 0 ms 2024-12-09T01:03:15,840 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:15,840 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:15,840 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5fdfc6cd0e1d087d33da2039f5cfb09e: Waiting for close lock at 1733706195839Disabling compacts and flushes for region at 1733706195839Disabling writes for close at 1733706195840 (+1 ms)Writing region close event to WAL at 1733706195840Closed at 1733706195840 2024-12-09T01:03:15,844 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:15,845 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733706195844"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706195844"}]},"ts":"1733706195844"} 2024-12-09T01:03:15,849 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:15,850 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:15,851 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706195850"}]},"ts":"1733706195850"} 2024-12-09T01:03:15,854 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-09T01:03:15,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5fdfc6cd0e1d087d33da2039f5cfb09e, ASSIGN}] 2024-12-09T01:03:15,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5fdfc6cd0e1d087d33da2039f5cfb09e, ASSIGN 2024-12-09T01:03:15,857 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5fdfc6cd0e1d087d33da2039f5cfb09e, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:03:16,008 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5fdfc6cd0e1d087d33da2039f5cfb09e, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:16,014 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5fdfc6cd0e1d087d33da2039f5cfb09e, ASSIGN because future has completed 2024-12-09T01:03:16,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fdfc6cd0e1d087d33da2039f5cfb09e, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:16,182 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:16,183 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 5fdfc6cd0e1d087d33da2039f5cfb09e, NAME => 'testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:16,183 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,183 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:16,183 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,183 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,186 INFO [StoreOpener-5fdfc6cd0e1d087d33da2039f5cfb09e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,189 INFO [StoreOpener-5fdfc6cd0e1d087d33da2039f5cfb09e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5fdfc6cd0e1d087d33da2039f5cfb09e columnFamilyName testFamily 2024-12-09T01:03:16,189 DEBUG [StoreOpener-5fdfc6cd0e1d087d33da2039f5cfb09e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:16,190 INFO [StoreOpener-5fdfc6cd0e1d087d33da2039f5cfb09e-1 {}] regionserver.HStore(327): Store=5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:16,191 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,192 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,193 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,193 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,193 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,196 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,200 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:16,200 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 5fdfc6cd0e1d087d33da2039f5cfb09e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72908236, jitterRate=0.08641737699508667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:16,201 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:03:16,201 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 5fdfc6cd0e1d087d33da2039f5cfb09e: Running coprocessor pre-open hook at 1733706196184Writing region info on filesystem at 1733706196184Initializing all the Stores at 1733706196185 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706196186 (+1 ms)Cleaning up temporary data from old regions at 1733706196193 (+7 ms)Running coprocessor post-open hooks at 1733706196201 (+8 ms)Region opened successfully at 1733706196201 2024-12-09T01:03:16,202 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., pid=15, masterSystemTime=1733706196170 2024-12-09T01:03:16,205 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:16,205 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:03:16,206 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5fdfc6cd0e1d087d33da2039f5cfb09e, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:16,209 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5fdfc6cd0e1d087d33da2039f5cfb09e, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:16,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T01:03:16,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5fdfc6cd0e1d087d33da2039f5cfb09e, server=1bae0942de96,46191,1733706140132 in 196 msec 2024-12-09T01:03:16,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T01:03:16,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5fdfc6cd0e1d087d33da2039f5cfb09e, ASSIGN in 360 msec 2024-12-09T01:03:16,218 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:16,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706196218"}]},"ts":"1733706196218"} 2024-12-09T01:03:16,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-09T01:03:16,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:16,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2640 sec 2024-12-09T01:03:16,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-09T01:03:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:03:17,987 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-09T01:03:17,987 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-09T01:03:17,987 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:17,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-09T01:03:17,995 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:17,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-09T01:03:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:18,014 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:18,014 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-09T01:03:18,018 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T01:03:18,021 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T01:03:18,023 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T01:03:18,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=161894076348629017 2024-12-09T01:03:18,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706201023, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:03:18,028 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T01:03:18,029 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T01:03:18,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:03:18,031 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:03:18,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=161894076348629019 2024-12-09T01:03:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706201031, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:03:18,035 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:03:18,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=161894076348629020 2024-12-09T01:03:18,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706201034, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:03:18,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:03:18,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=161894076348629021 2024-12-09T01:03:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:43206 deadline: 1733706201039, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T01:03:18,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T01:03:18,052 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=322 (was 322), ProcessCount=11 (was 11), AvailableMemoryMB=16345 (was 16350) 2024-12-09T01:03:18,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=322, ProcessCount=11, AvailableMemoryMB=16344 2024-12-09T01:03:18,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-09T01:03:18,066 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:18,066 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:18,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-09T01:03:18,068 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T01:03:18,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741843_1019 (size=418) 2024-12-09T01:03:18,122 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:03:18,486 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe65eafd217d2039389d32c4571fdf80, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:18,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741844_1020 (size=69) 2024-12-09T01:03:18,901 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:18,902 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing fe65eafd217d2039389d32c4571fdf80, disabling compactions & flushes 2024-12-09T01:03:18,902 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:18,902 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:18,902 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. after waiting 0 ms 2024-12-09T01:03:18,902 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:18,903 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:18,903 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe65eafd217d2039389d32c4571fdf80: Waiting for close lock at 1733706198902Disabling compacts and flushes for region at 1733706198902Disabling writes for close at 1733706198902Writing region close event to WAL at 1733706198902Closed at 1733706198902 2024-12-09T01:03:18,906 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:18,907 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733706198907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706198907"}]},"ts":"1733706198907"} 2024-12-09T01:03:18,912 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:18,913 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:18,914 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706198913"}]},"ts":"1733706198913"} 2024-12-09T01:03:18,916 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T01:03:18,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=fe65eafd217d2039389d32c4571fdf80, ASSIGN}] 2024-12-09T01:03:18,918 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=fe65eafd217d2039389d32c4571fdf80, ASSIGN 2024-12-09T01:03:18,920 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=fe65eafd217d2039389d32c4571fdf80, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:19,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=fe65eafd217d2039389d32c4571fdf80, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T01:03:19,078 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=fe65eafd217d2039389d32c4571fdf80, ASSIGN because future has completed 2024-12-09T01:03:19,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe65eafd217d2039389d32c4571fdf80, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:19,243 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:19,243 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => fe65eafd217d2039389d32c4571fdf80, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:19,244 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,244 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:19,244 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,244 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,246 INFO [StoreOpener-fe65eafd217d2039389d32c4571fdf80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,248 INFO [StoreOpener-fe65eafd217d2039389d32c4571fdf80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe65eafd217d2039389d32c4571fdf80 columnFamilyName testFamily 2024-12-09T01:03:19,248 DEBUG [StoreOpener-fe65eafd217d2039389d32c4571fdf80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:19,249 INFO [StoreOpener-fe65eafd217d2039389d32c4571fdf80-1 {}] regionserver.HStore(327): Store=fe65eafd217d2039389d32c4571fdf80/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:19,249 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,250 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,250 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,251 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,251 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,253 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,256 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:19,256 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened fe65eafd217d2039389d32c4571fdf80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71776142, jitterRate=0.0695478618144989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:19,256 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:03:19,257 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for fe65eafd217d2039389d32c4571fdf80: Running coprocessor pre-open hook at 1733706199244Writing region info on filesystem at 1733706199244Initializing all the Stores at 1733706199246 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706199246Cleaning up temporary data from old regions at 1733706199251 (+5 ms)Running coprocessor post-open hooks at 1733706199256 (+5 ms)Region opened successfully at 1733706199257 (+1 ms) 2024-12-09T01:03:19,258 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., pid=18, masterSystemTime=1733706199233 2024-12-09T01:03:19,261 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:19,261 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:03:19,262 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=fe65eafd217d2039389d32c4571fdf80, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:19,264 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe65eafd217d2039389d32c4571fdf80, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:19,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T01:03:19,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure fe65eafd217d2039389d32c4571fdf80, server=1bae0942de96,46191,1733706140132 in 188 msec 2024-12-09T01:03:19,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T01:03:19,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=fe65eafd217d2039389d32c4571fdf80, ASSIGN in 351 msec 2024-12-09T01:03:19,272 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:19,272 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706199272"}]},"ts":"1733706199272"} 2024-12-09T01:03:19,275 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T01:03:19,276 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:19,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2130 sec 2024-12-09T01:03:19,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-09T01:03:19,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:19,459 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-09T01:03:19,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T01:03:21,095 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-09T01:03:21,096 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T01:03:21,096 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:21,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T01:03:21,105 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:21,105 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-09T01:03:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:21,121 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:21,121 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T01:03:21,124 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T01:03:21,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T01:03:21,687 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:22,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-09T01:03:22,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-09T01:03:22,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:03:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-09T01:03:22,719 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-09T01:03:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:23,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T01:03:24,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=973ms 2024-12-09T01:03:24,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=981ms, rpcTimeout=973ms 2024-12-09T01:03:24,133 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:03:24,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=296 (was 322), ProcessCount=11 (was 11), AvailableMemoryMB=16338 (was 16344) 2024-12-09T01:03:24,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=296, ProcessCount=11, AvailableMemoryMB=16337 2024-12-09T01:03:24,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-09T01:03:24,164 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:24,164 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:24,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-09T01:03:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:03:24,165 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741845_1021 (size=416) 2024-12-09T01:03:24,582 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c6d4fa3aa54648498c3fb38c9152410, NAME => 'testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741846_1022 (size=67) 2024-12-09T01:03:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-09T01:03:24,995 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:24,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6c6d4fa3aa54648498c3fb38c9152410, disabling compactions & flushes 2024-12-09T01:03:24,996 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:24,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:24,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. after waiting 0 ms 2024-12-09T01:03:24,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:24,996 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:24,997 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c6d4fa3aa54648498c3fb38c9152410: Waiting for close lock at 1733706204995Disabling compacts and flushes for region at 1733706204995Disabling writes for close at 1733706204996 (+1 ms)Writing region close event to WAL at 1733706204996Closed at 1733706204996 2024-12-09T01:03:25,000 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:25,000 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733706205000"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706205000"}]},"ts":"1733706205000"} 2024-12-09T01:03:25,004 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:25,005 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:25,006 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706205006"}]},"ts":"1733706205006"} 2024-12-09T01:03:25,009 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T01:03:25,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=6c6d4fa3aa54648498c3fb38c9152410, ASSIGN}] 2024-12-09T01:03:25,011 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=6c6d4fa3aa54648498c3fb38c9152410, ASSIGN 2024-12-09T01:03:25,012 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=6c6d4fa3aa54648498c3fb38c9152410, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:25,163 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6c6d4fa3aa54648498c3fb38c9152410, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:25,168 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=6c6d4fa3aa54648498c3fb38c9152410, ASSIGN because future has completed 2024-12-09T01:03:25,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6d4fa3aa54648498c3fb38c9152410, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:25,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:03:25,336 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:25,337 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c6d4fa3aa54648498c3fb38c9152410, NAME => 'testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:25,338 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,338 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:25,338 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,338 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,340 INFO [StoreOpener-6c6d4fa3aa54648498c3fb38c9152410-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,342 INFO [StoreOpener-6c6d4fa3aa54648498c3fb38c9152410-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c6d4fa3aa54648498c3fb38c9152410 columnFamilyName testFamily 2024-12-09T01:03:25,342 DEBUG [StoreOpener-6c6d4fa3aa54648498c3fb38c9152410-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:25,343 INFO [StoreOpener-6c6d4fa3aa54648498c3fb38c9152410-1 {}] regionserver.HStore(327): Store=6c6d4fa3aa54648498c3fb38c9152410/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:25,343 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,344 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,345 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,346 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,346 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,348 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,351 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:25,352 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 6c6d4fa3aa54648498c3fb38c9152410; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67369082, jitterRate=0.0038775503635406494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:25,352 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:03:25,354 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 6c6d4fa3aa54648498c3fb38c9152410: Running coprocessor pre-open hook at 1733706205338Writing region info on filesystem at 1733706205338Initializing all the Stores at 1733706205340 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706205340Cleaning up temporary data from old regions at 1733706205346 (+6 ms)Running coprocessor post-open hooks at 1733706205353 (+7 ms)Region opened successfully at 1733706205354 (+1 ms) 2024-12-09T01:03:25,355 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410., pid=21, masterSystemTime=1733706205324 2024-12-09T01:03:25,358 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:25,358 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:03:25,359 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=6c6d4fa3aa54648498c3fb38c9152410, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:25,362 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c6d4fa3aa54648498c3fb38c9152410, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:25,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-09T01:03:25,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c6d4fa3aa54648498c3fb38c9152410, server=1bae0942de96,46191,1733706140132 in 195 msec 2024-12-09T01:03:25,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T01:03:25,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=6c6d4fa3aa54648498c3fb38c9152410, ASSIGN in 359 msec 2024-12-09T01:03:25,373 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:25,373 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706205373"}]},"ts":"1733706205373"} 2024-12-09T01:03:25,376 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T01:03:25,377 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:25,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2180 sec 2024-12-09T01:03:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:03:27,196 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-09T01:03:27,197 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T01:03:27,197 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:27,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T01:03:27,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:27,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-09T01:03:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:27,220 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:27,220 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T01:03:27,223 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T01:03:27,225 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T01:03:27,226 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T01:03:27,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T01:03:28,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-09T01:03:29,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-09T01:03:29,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:30,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629032' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T01:03:30,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43206 deadline: 1733706211233, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629032' does not exist 2024-12-09T01:03:30,239 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1bae0942de96,46191,1733706140132 for closing scanner id = 161894076348629032 for 6c6d4fa3aa54648498c3fb38c9152410 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629032' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:03:30,244 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 296), ProcessCount=11 (was 11), AvailableMemoryMB=16334 (was 16337) 2024-12-09T01:03:30,253 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=16334 2024-12-09T01:03:30,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-09T01:03:30,258 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:30,259 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:30,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-09T01:03:30,260 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T01:03:30,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741847_1023 (size=411) 2024-12-09T01:03:30,672 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29ad3b85d9009d8733d9b57c8cdf387f, NAME => 'testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:30,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741848_1024 (size=62) 2024-12-09T01:03:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-09T01:03:30,840 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:31,086 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:31,086 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 29ad3b85d9009d8733d9b57c8cdf387f, disabling compactions & flushes 2024-12-09T01:03:31,087 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,087 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,087 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. after waiting 0 ms 2024-12-09T01:03:31,087 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,087 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,087 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29ad3b85d9009d8733d9b57c8cdf387f: Waiting for close lock at 1733706211086Disabling compacts and flushes for region at 1733706211086Disabling writes for close at 1733706211087 (+1 ms)Writing region close event to WAL at 1733706211087Closed at 1733706211087 2024-12-09T01:03:31,130 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:31,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733706211130"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706211130"}]},"ts":"1733706211130"} 2024-12-09T01:03:31,135 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:31,137 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:31,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706211137"}]},"ts":"1733706211137"} 2024-12-09T01:03:31,140 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T01:03:31,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=29ad3b85d9009d8733d9b57c8cdf387f, ASSIGN}] 2024-12-09T01:03:31,142 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=29ad3b85d9009d8733d9b57c8cdf387f, ASSIGN 2024-12-09T01:03:31,144 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=29ad3b85d9009d8733d9b57c8cdf387f, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T01:03:31,295 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=29ad3b85d9009d8733d9b57c8cdf387f, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:31,301 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=29ad3b85d9009d8733d9b57c8cdf387f, ASSIGN because future has completed 2024-12-09T01:03:31,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29ad3b85d9009d8733d9b57c8cdf387f, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:31,470 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,470 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 29ad3b85d9009d8733d9b57c8cdf387f, NAME => 'testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:31,471 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,471 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:31,471 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,471 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,473 INFO [StoreOpener-29ad3b85d9009d8733d9b57c8cdf387f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,475 INFO [StoreOpener-29ad3b85d9009d8733d9b57c8cdf387f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29ad3b85d9009d8733d9b57c8cdf387f columnFamilyName testFamily 2024-12-09T01:03:31,475 DEBUG [StoreOpener-29ad3b85d9009d8733d9b57c8cdf387f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:31,476 INFO [StoreOpener-29ad3b85d9009d8733d9b57c8cdf387f-1 {}] regionserver.HStore(327): Store=29ad3b85d9009d8733d9b57c8cdf387f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:31,476 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,478 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,478 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,479 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,479 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,482 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,485 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:31,485 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 29ad3b85d9009d8733d9b57c8cdf387f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63309396, jitterRate=-0.05661648511886597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:31,485 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:03:31,486 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 29ad3b85d9009d8733d9b57c8cdf387f: Running coprocessor pre-open hook at 1733706211471Writing region info on filesystem at 1733706211471Initializing all the Stores at 1733706211473 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706211473Cleaning up temporary data from old regions at 1733706211479 (+6 ms)Running coprocessor post-open hooks at 1733706211485 (+6 ms)Region opened successfully at 1733706211486 (+1 ms) 2024-12-09T01:03:31,487 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f., pid=24, masterSystemTime=1733706211458 2024-12-09T01:03:31,489 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,490 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:03:31,490 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=29ad3b85d9009d8733d9b57c8cdf387f, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:31,493 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29ad3b85d9009d8733d9b57c8cdf387f, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:31,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T01:03:31,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 29ad3b85d9009d8733d9b57c8cdf387f, server=1bae0942de96,46191,1733706140132 in 192 msec 2024-12-09T01:03:31,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-09T01:03:31,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=29ad3b85d9009d8733d9b57c8cdf387f, ASSIGN in 357 msec 2024-12-09T01:03:31,502 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:31,502 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706211502"}]},"ts":"1733706211502"} 2024-12-09T01:03:31,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T01:03:31,505 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:31,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2510 sec 2024-12-09T01:03:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T01:03:33,284 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-09T01:03:33,284 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T01:03:33,284 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:33,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T01:03:33,291 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:33,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-09T01:03:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:33,306 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:33,306 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T01:03:33,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T01:03:33,313 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T01:03:33,314 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T01:03:33,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T01:03:33,393 INFO [master/1bae0942de96:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T01:03:33,393 INFO [master/1bae0942de96:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T01:03:34,722 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-09T01:03:36,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629036' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T01:03:36,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43206 deadline: 1733706217328, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629036' does not exist 2024-12-09T01:03:36,330 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1bae0942de96,46191,1733706140132 for closing scanner id = 161894076348629036 for 29ad3b85d9009d8733d9b57c8cdf387f of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629036' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:03:36,338 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=16324 (was 16334) 2024-12-09T01:03:36,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=16324 2024-12-09T01:03:36,351 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T01:03:36,372 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:36,372 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:36,372 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-09T01:03:36,374 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:03:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741849_1025 (size=419) 2024-12-09T01:03:36,793 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 635b2e88cb0de9431907c0b036d819ae, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:36,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741850_1026 (size=70) 2024-12-09T01:03:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-09T01:03:36,986 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:37,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:37,208 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 635b2e88cb0de9431907c0b036d819ae, disabling compactions & flushes 2024-12-09T01:03:37,208 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,208 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,208 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. after waiting 0 ms 2024-12-09T01:03:37,209 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,209 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,209 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 635b2e88cb0de9431907c0b036d819ae: Waiting for close lock at 1733706217207Disabling compacts and flushes for region at 1733706217207Disabling writes for close at 1733706217208 (+1 ms)Writing region close event to WAL at 1733706217209 (+1 ms)Closed at 1733706217209 2024-12-09T01:03:37,212 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:37,212 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733706217212"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706217212"}]},"ts":"1733706217212"} 2024-12-09T01:03:37,215 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:37,217 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:37,217 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706217217"}]},"ts":"1733706217217"} 2024-12-09T01:03:37,219 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T01:03:37,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=635b2e88cb0de9431907c0b036d819ae, ASSIGN}] 2024-12-09T01:03:37,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=635b2e88cb0de9431907c0b036d819ae, ASSIGN 2024-12-09T01:03:37,222 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=635b2e88cb0de9431907c0b036d819ae, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:37,373 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=635b2e88cb0de9431907c0b036d819ae, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:37,377 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=635b2e88cb0de9431907c0b036d819ae, ASSIGN because future has completed 2024-12-09T01:03:37,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 635b2e88cb0de9431907c0b036d819ae, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:03:37,542 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,542 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 635b2e88cb0de9431907c0b036d819ae, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:37,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:37,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,547 INFO [StoreOpener-635b2e88cb0de9431907c0b036d819ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,551 INFO [StoreOpener-635b2e88cb0de9431907c0b036d819ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 635b2e88cb0de9431907c0b036d819ae columnFamilyName testFamily 2024-12-09T01:03:37,551 DEBUG [StoreOpener-635b2e88cb0de9431907c0b036d819ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:37,552 INFO [StoreOpener-635b2e88cb0de9431907c0b036d819ae-1 {}] regionserver.HStore(327): Store=635b2e88cb0de9431907c0b036d819ae/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:37,552 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,553 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,560 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:37,561 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 635b2e88cb0de9431907c0b036d819ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67987600, jitterRate=0.013094186782836914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:37,561 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:03:37,561 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 635b2e88cb0de9431907c0b036d819ae: Running coprocessor pre-open hook at 1733706217544Writing region info on filesystem at 1733706217544Initializing all the Stores at 1733706217546 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706217547 (+1 ms)Cleaning up temporary data from old regions at 1733706217554 (+7 ms)Running coprocessor post-open hooks at 1733706217561 (+7 ms)Region opened successfully at 1733706217561 2024-12-09T01:03:37,562 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., pid=27, masterSystemTime=1733706217531 2024-12-09T01:03:37,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,565 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:03:37,566 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=635b2e88cb0de9431907c0b036d819ae, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:37,568 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 635b2e88cb0de9431907c0b036d819ae, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:37,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T01:03:37,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 635b2e88cb0de9431907c0b036d819ae, server=1bae0942de96,46191,1733706140132 in 192 msec 2024-12-09T01:03:37,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T01:03:37,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=635b2e88cb0de9431907c0b036d819ae, ASSIGN in 353 msec 2024-12-09T01:03:37,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:37,576 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706217576"}]},"ts":"1733706217576"} 2024-12-09T01:03:37,578 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T01:03:37,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:37,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2290 sec 2024-12-09T01:03:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:03:39,396 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-09T01:03:39,396 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T01:03:39,396 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:39,406 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T01:03:39,406 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:39,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T01:03:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:39,419 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:39,419 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T01:03:39,422 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T01:03:39,423 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T01:03:39,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-09T01:03:39,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:39,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T01:03:39,457 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:40,725 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-09T01:03:43,047 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:45,422 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-09T01:03:45,422 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6000ms, rpcTimeout=5999ms 2024-12-09T01:03:45,422 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:03:45,431 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=210 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=16315 (was 16324) 2024-12-09T01:03:45,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=210, ProcessCount=11, AvailableMemoryMB=16315 2024-12-09T01:03:45,442 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:45,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-09T01:03:45,446 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:45,446 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:45,446 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-09T01:03:45,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T01:03:45,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741851_1027 (size=407) 2024-12-09T01:03:45,859 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 674f2f03dd0bb0450e3ead2e9b98b5fa, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:45,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741852_1028 (size=58) 2024-12-09T01:03:45,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-09T01:03:46,273 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:46,273 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 674f2f03dd0bb0450e3ead2e9b98b5fa, disabling compactions & flushes 2024-12-09T01:03:46,273 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,273 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,274 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. after waiting 0 ms 2024-12-09T01:03:46,274 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,274 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,274 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 674f2f03dd0bb0450e3ead2e9b98b5fa: Waiting for close lock at 1733706226273Disabling compacts and flushes for region at 1733706226273Disabling writes for close at 1733706226274 (+1 ms)Writing region close event to WAL at 1733706226274Closed at 1733706226274 2024-12-09T01:03:46,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:46,279 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733706226278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706226278"}]},"ts":"1733706226278"} 2024-12-09T01:03:46,284 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:46,285 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:46,285 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706226285"}]},"ts":"1733706226285"} 2024-12-09T01:03:46,287 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T01:03:46,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=674f2f03dd0bb0450e3ead2e9b98b5fa, ASSIGN}] 2024-12-09T01:03:46,289 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=674f2f03dd0bb0450e3ead2e9b98b5fa, ASSIGN 2024-12-09T01:03:46,290 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=674f2f03dd0bb0450e3ead2e9b98b5fa, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:46,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=674f2f03dd0bb0450e3ead2e9b98b5fa, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:46,448 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=674f2f03dd0bb0450e3ead2e9b98b5fa, ASSIGN because future has completed 2024-12-09T01:03:46,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 674f2f03dd0bb0450e3ead2e9b98b5fa, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T01:03:46,610 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,610 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 674f2f03dd0bb0450e3ead2e9b98b5fa, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:46,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:46,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,614 INFO [StoreOpener-674f2f03dd0bb0450e3ead2e9b98b5fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,616 INFO [StoreOpener-674f2f03dd0bb0450e3ead2e9b98b5fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 674f2f03dd0bb0450e3ead2e9b98b5fa columnFamilyName testFamily 2024-12-09T01:03:46,616 DEBUG [StoreOpener-674f2f03dd0bb0450e3ead2e9b98b5fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:46,617 INFO [StoreOpener-674f2f03dd0bb0450e3ead2e9b98b5fa-1 {}] regionserver.HStore(327): Store=674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:46,617 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,619 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,619 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,623 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,626 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:46,627 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 674f2f03dd0bb0450e3ead2e9b98b5fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69764794, jitterRate=0.039576441049575806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:46,627 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:03:46,628 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 674f2f03dd0bb0450e3ead2e9b98b5fa: Running coprocessor pre-open hook at 1733706226611Writing region info on filesystem at 1733706226611Initializing all the Stores at 1733706226613 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706226613Cleaning up temporary data from old regions at 1733706226620 (+7 ms)Running coprocessor post-open hooks at 1733706226627 (+7 ms)Region opened successfully at 1733706226628 (+1 ms) 2024-12-09T01:03:46,629 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa., pid=30, masterSystemTime=1733706226603 2024-12-09T01:03:46,632 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,632 INFO [RS_OPEN_PRIORITY_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:03:46,633 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=674f2f03dd0bb0450e3ead2e9b98b5fa, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:46,636 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 674f2f03dd0bb0450e3ead2e9b98b5fa, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:46,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T01:03:46,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 674f2f03dd0bb0450e3ead2e9b98b5fa, server=1bae0942de96,46191,1733706140132 in 189 msec 2024-12-09T01:03:46,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-09T01:03:46,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=674f2f03dd0bb0450e3ead2e9b98b5fa, ASSIGN in 353 msec 2024-12-09T01:03:46,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:46,644 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706226644"}]},"ts":"1733706226644"} 2024-12-09T01:03:46,646 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T01:03:46,647 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:46,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2050 sec 2024-12-09T01:03:46,727 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-09T01:03:48,122 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:03:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T01:03:48,484 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-09T01:03:48,484 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T01:03:48,484 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:48,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T01:03:48,491 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:48,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-09T01:03:48,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:48,505 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:48,505 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T01:03:48,508 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T01:03:48,510 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T01:03:48,511 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T01:03:48,512 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T01:03:49,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-09T01:03:49,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:52,116 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:03:53,355 DEBUG [master/1bae0942de96:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T01:03:54,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629044' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T01:03:54,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46191 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:43206 deadline: 1733706240520, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629044' does not exist 2024-12-09T01:03:54,521 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 1bae0942de96,46191,1733706140132 for closing scanner id = 161894076348629044 for 674f2f03dd0bb0450e3ead2e9b98b5fa of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '161894076348629044' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:03:54,526 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 210), ProcessCount=11 (was 11), AvailableMemoryMB=16313 (was 16315) 2024-12-09T01:03:54,535 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=16313 2024-12-09T01:03:54,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:03:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T01:03:54,540 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:03:54,540 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:54,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-09T01:03:54,541 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:03:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T01:03:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741853_1029 (size=423) 2024-12-09T01:03:54,955 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b36dedecc3c4f56c9617b0a8a6b58ae1, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0 2024-12-09T01:03:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741854_1030 (size=74) 2024-12-09T01:03:55,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-09T01:03:55,368 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:55,368 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing b36dedecc3c4f56c9617b0a8a6b58ae1, disabling compactions & flushes 2024-12-09T01:03:55,369 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,369 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,369 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. after waiting 0 ms 2024-12-09T01:03:55,369 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,369 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,370 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b36dedecc3c4f56c9617b0a8a6b58ae1: Waiting for close lock at 1733706235368Disabling compacts and flushes for region at 1733706235368Disabling writes for close at 1733706235369 (+1 ms)Writing region close event to WAL at 1733706235369Closed at 1733706235369 2024-12-09T01:03:55,375 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:03:55,375 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733706235375"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733706235375"}]},"ts":"1733706235375"} 2024-12-09T01:03:55,377 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:03:55,379 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:03:55,379 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706235379"}]},"ts":"1733706235379"} 2024-12-09T01:03:55,381 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T01:03:55,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=b36dedecc3c4f56c9617b0a8a6b58ae1, ASSIGN}] 2024-12-09T01:03:55,382 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=b36dedecc3c4f56c9617b0a8a6b58ae1, ASSIGN 2024-12-09T01:03:55,383 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=b36dedecc3c4f56c9617b0a8a6b58ae1, ASSIGN; state=OFFLINE, location=1bae0942de96,46191,1733706140132; forceNewPlan=false, retain=false 2024-12-09T01:03:55,535 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=b36dedecc3c4f56c9617b0a8a6b58ae1, regionState=OPENING, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:55,540 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=b36dedecc3c4f56c9617b0a8a6b58ae1, ASSIGN because future has completed 2024-12-09T01:03:55,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure b36dedecc3c4f56c9617b0a8a6b58ae1, server=1bae0942de96,46191,1733706140132}] 2024-12-09T01:03:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T01:03:55,701 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,702 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => b36dedecc3c4f56c9617b0a8a6b58ae1, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:03:55,702 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,702 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:03:55,703 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,703 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,705 INFO [StoreOpener-b36dedecc3c4f56c9617b0a8a6b58ae1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,707 INFO [StoreOpener-b36dedecc3c4f56c9617b0a8a6b58ae1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b36dedecc3c4f56c9617b0a8a6b58ae1 columnFamilyName testFamily 2024-12-09T01:03:55,707 DEBUG [StoreOpener-b36dedecc3c4f56c9617b0a8a6b58ae1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:03:55,707 INFO [StoreOpener-b36dedecc3c4f56c9617b0a8a6b58ae1-1 {}] regionserver.HStore(327): Store=b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:03:55,708 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,709 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,709 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,709 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,709 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,711 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,714 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:03:55,714 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened b36dedecc3c4f56c9617b0a8a6b58ae1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60399249, jitterRate=-0.09998105466365814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:03:55,714 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:03:55,715 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for b36dedecc3c4f56c9617b0a8a6b58ae1: Running coprocessor pre-open hook at 1733706235703Writing region info on filesystem at 1733706235703Initializing all the Stores at 1733706235704 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733706235704Cleaning up temporary data from old regions at 1733706235710 (+6 ms)Running coprocessor post-open hooks at 1733706235714 (+4 ms)Region opened successfully at 1733706235715 (+1 ms) 2024-12-09T01:03:55,716 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., pid=33, masterSystemTime=1733706235695 2024-12-09T01:03:55,718 DEBUG [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,719 INFO [RS_OPEN_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:03:55,719 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=b36dedecc3c4f56c9617b0a8a6b58ae1, regionState=OPEN, openSeqNum=2, regionLocation=1bae0942de96,46191,1733706140132 2024-12-09T01:03:55,722 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure b36dedecc3c4f56c9617b0a8a6b58ae1, server=1bae0942de96,46191,1733706140132 because future has completed 2024-12-09T01:03:55,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-09T01:03:55,727 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure b36dedecc3c4f56c9617b0a8a6b58ae1, server=1bae0942de96,46191,1733706140132 in 183 msec 2024-12-09T01:03:55,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T01:03:55,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=b36dedecc3c4f56c9617b0a8a6b58ae1, ASSIGN in 345 msec 2024-12-09T01:03:55,730 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:03:55,730 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733706235730"}]},"ts":"1733706235730"} 2024-12-09T01:03:55,732 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T01:03:55,733 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:03:55,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1960 sec 2024-12-09T01:03:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T01:03:57,566 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-09T01:03:57,566 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T01:03:57,566 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:57,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T01:03:57,575 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:03:57,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T01:03:57,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:57,591 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T01:03:57,591 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T01:03:57,595 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T01:03:57,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T01:03:58,602 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-12-09T01:03:58,603 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-12-09T01:03:58,603 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:03:58,730 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-09T01:03:59,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-09T01:03:59,456 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T01:03:59,456 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T01:03:59,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2] 2024-12-09T01:03:59,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T01:04:00,603 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=1bae0942de96:46191 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=984ms, rpcTimeout=975ms 2024-12-09T01:04:00,603 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=984ms, rpcTimeout=975ms 2024-12-09T01:04:00,603 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., hostname=1bae0942de96,46191,1733706140132, seqNum=2 from cache 2024-12-09T01:04:00,613 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=163 (was 177), ProcessCount=11 (was 11), AvailableMemoryMB=16303 (was 16313) 2024-12-09T01:04:00,613 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:04:00,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:04:00,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:04:00,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T01:04:00,619 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:04:00,620 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:04:00,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:04:00,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T01:04:00,620 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1546128933, stopped=false 2024-12-09T01:04:00,620 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1bae0942de96,34071,1733706138987 2024-12-09T01:04:00,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:04:00,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:04:00,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:04:00,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:04:00,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:04:00,677 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:04:00,677 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:04:00,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:04:00,677 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:04:00,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,678 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1bae0942de96,46191,1733706140132' ***** 2024-12-09T01:04:00,678 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:04:00,678 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:04:00,678 INFO [RS:0;1bae0942de96:46191 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:04:00,678 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:04:00,678 INFO [RS:0;1bae0942de96:46191 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:04:00,678 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 674f2f03dd0bb0450e3ead2e9b98b5fa 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for aba63087cedfb13adaecd45bf828b8ff 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 17a3f7298b74b7c3bf26405f5118df93 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c6d4fa3aa54648498c3fb38c9152410 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 29ad3b85d9009d8733d9b57c8cdf387f 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 5fdfc6cd0e1d087d33da2039f5cfb09e 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:00,679 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 674f2f03dd0bb0450e3ead2e9b98b5fa, disabling compactions & flushes 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(959): stopping server 1bae0942de96,46191,1733706140132 2024-12-09T01:04:00,679 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:04:00,679 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:04:00,679 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. after waiting 0 ms 2024-12-09T01:04:00,679 INFO [RS:0;1bae0942de96:46191 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1bae0942de96:46191. 2024-12-09T01:04:00,679 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:04:00,680 DEBUG [RS:0;1bae0942de96:46191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:04:00,680 DEBUG [RS:0;1bae0942de96:46191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:00,680 INFO [RS:0;1bae0942de96:46191 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:04:00,680 INFO [RS:0;1bae0942de96:46191 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:04:00,680 INFO [RS:0;1bae0942de96:46191 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:04:00,680 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T01:04:00,680 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-09T01:04:00,680 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:04:00,681 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:04:00,681 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:04:00,681 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 674f2f03dd0bb0450e3ead2e9b98b5fa, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, aba63087cedfb13adaecd45bf828b8ff, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:00,681 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:04:00,681 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:04:00,683 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 674f2f03dd0bb0450e3ead2e9b98b5fa 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:00,683 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-09T01:04:00,731 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/.tmp/testFamily/592c94afbe9546bb9f930e7b330917c3 is 61, key is row-0/testFamily:testQualifier/1733706228496/Put/seqid=0 2024-12-09T01:04:00,735 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/info/a55250b7ca9e40d3878dcf54f73127b1 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93./info:regioninfo/1733706174829/Put/seqid=0 2024-12-09T01:04:00,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741855_1031 (size=5049) 2024-12-09T01:04:00,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741856_1032 (size=20564) 2024-12-09T01:04:00,740 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/.tmp/testFamily/592c94afbe9546bb9f930e7b330917c3 2024-12-09T01:04:00,778 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/.tmp/testFamily/592c94afbe9546bb9f930e7b330917c3 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily/592c94afbe9546bb9f930e7b330917c3 2024-12-09T01:04:00,786 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily/592c94afbe9546bb9f930e7b330917c3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:00,793 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 674f2f03dd0bb0450e3ead2e9b98b5fa in 110ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:00,800 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNext/674f2f03dd0bb0450e3ead2e9b98b5fa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:00,802 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:04:00,802 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 674f2f03dd0bb0450e3ead2e9b98b5fa: Waiting for close lock at 1733706240679Running coprocessor pre-close hooks at 1733706240679Disabling compacts and flushes for region at 1733706240679Disabling writes for close at 1733706240679Obtaining lock to block concurrent updates at 1733706240683 (+4 ms)Preparing flush snapshotting stores in 674f2f03dd0bb0450e3ead2e9b98b5fa at 1733706240683Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706240689 (+6 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. at 1733706240690 (+1 ms)Flushing 674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily: creating writer at 1733706240691 (+1 ms)Flushing 674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily: appending metadata at 1733706240726 (+35 ms)Flushing 674f2f03dd0bb0450e3ead2e9b98b5fa/testFamily: closing flushed file at 1733706240728 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57b5c690: reopening flushed file at 1733706240777 (+49 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 674f2f03dd0bb0450e3ead2e9b98b5fa in 110ms, sequenceid=8, compaction requested=false at 1733706240793 (+16 ms)Writing region close event to WAL at 1733706240795 (+2 ms)Running coprocessor post-close hooks at 1733706240801 (+6 ms)Closed at 1733706240802 (+1 ms) 2024-12-09T01:04:00,803 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733706225442.674f2f03dd0bb0450e3ead2e9b98b5fa. 2024-12-09T01:04:00,803 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aba63087cedfb13adaecd45bf828b8ff, disabling compactions & flushes 2024-12-09T01:04:00,803 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:04:00,803 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:04:00,803 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. after waiting 0 ms 2024-12-09T01:04:00,803 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:04:00,803 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing aba63087cedfb13adaecd45bf828b8ff 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:00,819 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/.tmp/testFamily/4f39758b58d64296bca530939f6b7cd0 is 61, key is row-0/testFamily:testQualifier/1733706179811/Put/seqid=0 2024-12-09T01:04:00,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741857_1033 (size=5049) 2024-12-09T01:04:00,881 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, aba63087cedfb13adaecd45bf828b8ff, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:01,081 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, aba63087cedfb13adaecd45bf828b8ff, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:01,122 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-09T01:04:01,143 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/info/a55250b7ca9e40d3878dcf54f73127b1 2024-12-09T01:04:01,174 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/ns/6b7312da64a643e0ba9f199dcfd1f855 is 43, key is default/ns:d/1733706173302/Put/seqid=0 2024-12-09T01:04:01,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741858_1034 (size=5153) 2024-12-09T01:04:01,204 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:04:01,225 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/.tmp/testFamily/4f39758b58d64296bca530939f6b7cd0 2024-12-09T01:04:01,234 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/.tmp/testFamily/4f39758b58d64296bca530939f6b7cd0 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/testFamily/4f39758b58d64296bca530939f6b7cd0 2024-12-09T01:04:01,241 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/testFamily/4f39758b58d64296bca530939f6b7cd0, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:01,242 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aba63087cedfb13adaecd45bf828b8ff in 439ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:01,247 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScanner/aba63087cedfb13adaecd45bf828b8ff/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:01,248 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aba63087cedfb13adaecd45bf828b8ff: Waiting for close lock at 1733706240803Running coprocessor pre-close hooks at 1733706240803Disabling compacts and flushes for region at 1733706240803Disabling writes for close at 1733706240803Obtaining lock to block concurrent updates at 1733706240803Preparing flush snapshotting stores in aba63087cedfb13adaecd45bf828b8ff at 1733706240803Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706240803Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. at 1733706240804 (+1 ms)Flushing aba63087cedfb13adaecd45bf828b8ff/testFamily: creating writer at 1733706240804Flushing aba63087cedfb13adaecd45bf828b8ff/testFamily: appending metadata at 1733706240818 (+14 ms)Flushing aba63087cedfb13adaecd45bf828b8ff/testFamily: closing flushed file at 1733706240818Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60020b1b: reopening flushed file at 1733706241233 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for aba63087cedfb13adaecd45bf828b8ff in 439ms, sequenceid=8, compaction requested=false at 1733706241242 (+9 ms)Writing region close event to WAL at 1733706241243 (+1 ms)Running coprocessor post-close hooks at 1733706241247 (+4 ms)Closed at 1733706241248 (+1 ms) 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733706176760.aba63087cedfb13adaecd45bf828b8ff. 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe65eafd217d2039389d32c4571fdf80, disabling compactions & flushes 2024-12-09T01:04:01,248 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. after waiting 0 ms 2024-12-09T01:04:01,248 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:04:01,248 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fe65eafd217d2039389d32c4571fdf80 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:01,263 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/.tmp/testFamily/ce4c2d1568f14387ab0b27c10514cf61 is 61, key is row-0/testFamily:testQualifier/1733706201110/Put/seqid=0 2024-12-09T01:04:01,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741859_1035 (size=5049) 2024-12-09T01:04:01,282 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:01,482 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:01,581 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/ns/6b7312da64a643e0ba9f199dcfd1f855 2024-12-09T01:04:01,608 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/table/ed225e108e9d4be8b5e9b1294aeb5a47 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733706174846/Put/seqid=0 2024-12-09T01:04:01,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741860_1036 (size=6699) 2024-12-09T01:04:01,635 INFO [regionserver/1bae0942de96:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T01:04:01,635 INFO [regionserver/1bae0942de96:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T01:04:01,669 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/.tmp/testFamily/ce4c2d1568f14387ab0b27c10514cf61 2024-12-09T01:04:01,678 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/.tmp/testFamily/ce4c2d1568f14387ab0b27c10514cf61 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/testFamily/ce4c2d1568f14387ab0b27c10514cf61 2024-12-09T01:04:01,682 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-09T01:04:01,682 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1325): Online Regions={fe65eafd217d2039389d32c4571fdf80=testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., 1588230740=hbase:meta,,1.1588230740, 17a3f7298b74b7c3bf26405f5118df93=testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93., 6c6d4fa3aa54648498c3fb38c9152410=testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410., 29ad3b85d9009d8733d9b57c8cdf387f=testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f., b36dedecc3c4f56c9617b0a8a6b58ae1=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., 68c0546b8ae3c545b7fd61a2ea54459d=hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., 5fdfc6cd0e1d087d33da2039f5cfb09e=testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., 635b2e88cb0de9431907c0b036d819ae=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.} 2024-12-09T01:04:01,683 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1, fe65eafd217d2039389d32c4571fdf80 2024-12-09T01:04:01,688 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/testFamily/ce4c2d1568f14387ab0b27c10514cf61, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:01,690 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe65eafd217d2039389d32c4571fdf80 in 441ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:01,695 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScanner/fe65eafd217d2039389d32c4571fdf80/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:01,696 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe65eafd217d2039389d32c4571fdf80: Waiting for close lock at 1733706241248Running coprocessor pre-close hooks at 1733706241248Disabling compacts and flushes for region at 1733706241248Disabling writes for close at 1733706241248Obtaining lock to block concurrent updates at 1733706241248Preparing flush snapshotting stores in fe65eafd217d2039389d32c4571fdf80 at 1733706241248Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706241249 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. at 1733706241249Flushing fe65eafd217d2039389d32c4571fdf80/testFamily: creating writer at 1733706241249Flushing fe65eafd217d2039389d32c4571fdf80/testFamily: appending metadata at 1733706241263 (+14 ms)Flushing fe65eafd217d2039389d32c4571fdf80/testFamily: closing flushed file at 1733706241263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a13e198: reopening flushed file at 1733706241677 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe65eafd217d2039389d32c4571fdf80 in 441ms, sequenceid=8, compaction requested=false at 1733706241690 (+13 ms)Writing region close event to WAL at 1733706241690Running coprocessor post-close hooks at 1733706241696 (+6 ms)Closed at 1733706241696 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733706198062.fe65eafd217d2039389d32c4571fdf80. 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17a3f7298b74b7c3bf26405f5118df93, disabling compactions & flushes 2024-12-09T01:04:01,697 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. after waiting 0 ms 2024-12-09T01:04:01,697 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:04:01,697 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 17a3f7298b74b7c3bf26405f5118df93 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:01,717 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/.tmp/testFamily/ab7d9417c4434642ae75cb09b54a76d7 is 61, key is row-0/testFamily:testQualifier/1733706176674/Put/seqid=0 2024-12-09T01:04:01,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741861_1037 (size=5049) 2024-12-09T01:04:01,883 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:02,015 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/table/ed225e108e9d4be8b5e9b1294aeb5a47 2024-12-09T01:04:02,030 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/info/a55250b7ca9e40d3878dcf54f73127b1 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/info/a55250b7ca9e40d3878dcf54f73127b1 2024-12-09T01:04:02,038 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/info/a55250b7ca9e40d3878dcf54f73127b1, entries=100, sequenceid=56, filesize=20.1 K 2024-12-09T01:04:02,039 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/ns/6b7312da64a643e0ba9f199dcfd1f855 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/ns/6b7312da64a643e0ba9f199dcfd1f855 2024-12-09T01:04:02,047 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/ns/6b7312da64a643e0ba9f199dcfd1f855, entries=2, sequenceid=56, filesize=5.0 K 2024-12-09T01:04:02,049 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/.tmp/table/ed225e108e9d4be8b5e9b1294aeb5a47 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/table/ed225e108e9d4be8b5e9b1294aeb5a47 2024-12-09T01:04:02,057 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/table/ed225e108e9d4be8b5e9b1294aeb5a47, entries=20, sequenceid=56, filesize=6.5 K 2024-12-09T01:04:02,058 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1377ms, sequenceid=56, compaction requested=false 2024-12-09T01:04:02,064 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-09T01:04:02,065 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:04:02,065 INFO [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:04:02,066 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733706240680Running coprocessor pre-close hooks at 1733706240680Disabling compacts and flushes for region at 1733706240680Disabling writes for close at 1733706240681 (+1 ms)Obtaining lock to block concurrent updates at 1733706240683 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733706240683Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733706240689 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733706240690 (+1 ms)Flushing 1588230740/info: creating writer at 1733706240691 (+1 ms)Flushing 1588230740/info: appending metadata at 1733706240732 (+41 ms)Flushing 1588230740/info: closing flushed file at 1733706240732Flushing 1588230740/ns: creating writer at 1733706241158 (+426 ms)Flushing 1588230740/ns: appending metadata at 1733706241173 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733706241173Flushing 1588230740/table: creating writer at 1733706241593 (+420 ms)Flushing 1588230740/table: appending metadata at 1733706241607 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733706241607Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d099dbd: reopening flushed file at 1733706242028 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49f0fd1a: reopening flushed file at 1733706242038 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@785264b2: reopening flushed file at 1733706242048 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1377ms, sequenceid=56, compaction requested=false at 1733706242058 (+10 ms)Writing region close event to WAL at 1733706242060 (+2 ms)Running coprocessor post-close hooks at 1733706242065 (+5 ms)Closed at 1733706242065 2024-12-09T01:04:02,066 DEBUG [RS_CLOSE_META-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T01:04:02,083 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 17a3f7298b74b7c3bf26405f5118df93, 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:02,122 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/.tmp/testFamily/ab7d9417c4434642ae75cb09b54a76d7 2024-12-09T01:04:02,133 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/.tmp/testFamily/ab7d9417c4434642ae75cb09b54a76d7 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/testFamily/ab7d9417c4434642ae75cb09b54a76d7 2024-12-09T01:04:02,140 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/testFamily/ab7d9417c4434642ae75cb09b54a76d7, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:02,141 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17a3f7298b74b7c3bf26405f5118df93 in 444ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:02,147 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/17a3f7298b74b7c3bf26405f5118df93/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:02,147 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17a3f7298b74b7c3bf26405f5118df93: Waiting for close lock at 1733706241697Running coprocessor pre-close hooks at 1733706241697Disabling compacts and flushes for region at 1733706241697Disabling writes for close at 1733706241697Obtaining lock to block concurrent updates at 1733706241697Preparing flush snapshotting stores in 17a3f7298b74b7c3bf26405f5118df93 at 1733706241697Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706241697Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. at 1733706241698 (+1 ms)Flushing 17a3f7298b74b7c3bf26405f5118df93/testFamily: creating writer at 1733706241698Flushing 17a3f7298b74b7c3bf26405f5118df93/testFamily: appending metadata at 1733706241716 (+18 ms)Flushing 17a3f7298b74b7c3bf26405f5118df93/testFamily: closing flushed file at 1733706241716Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a9887a2: reopening flushed file at 1733706242132 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17a3f7298b74b7c3bf26405f5118df93 in 444ms, sequenceid=8, compaction requested=false at 1733706242141 (+9 ms)Writing region close event to WAL at 1733706242142 (+1 ms)Running coprocessor post-close hooks at 1733706242147 (+5 ms)Closed at 1733706242147 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733706173579.17a3f7298b74b7c3bf26405f5118df93. 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c6d4fa3aa54648498c3fb38c9152410, disabling compactions & flushes 2024-12-09T01:04:02,148 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. after waiting 0 ms 2024-12-09T01:04:02,148 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:04:02,148 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6c6d4fa3aa54648498c3fb38c9152410 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:02,163 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/.tmp/testFamily/d57020a61d944f649d66703910bddbff is 61, key is row-0/testFamily:testQualifier/1733706207211/Put/seqid=0 2024-12-09T01:04:02,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741862_1038 (size=5049) 2024-12-09T01:04:02,283 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:02,484 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, 6c6d4fa3aa54648498c3fb38c9152410, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:02,571 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/.tmp/testFamily/d57020a61d944f649d66703910bddbff 2024-12-09T01:04:02,634 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/.tmp/testFamily/d57020a61d944f649d66703910bddbff as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/testFamily/d57020a61d944f649d66703910bddbff 2024-12-09T01:04:02,642 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/testFamily/d57020a61d944f649d66703910bddbff, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:02,643 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c6d4fa3aa54648498c3fb38c9152410 in 495ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:02,647 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNextAsync/6c6d4fa3aa54648498c3fb38c9152410/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:02,648 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:04:02,648 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c6d4fa3aa54648498c3fb38c9152410: Waiting for close lock at 1733706242148Running coprocessor pre-close hooks at 1733706242148Disabling compacts and flushes for region at 1733706242148Disabling writes for close at 1733706242148Obtaining lock to block concurrent updates at 1733706242148Preparing flush snapshotting stores in 6c6d4fa3aa54648498c3fb38c9152410 at 1733706242148Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706242149 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. at 1733706242149Flushing 6c6d4fa3aa54648498c3fb38c9152410/testFamily: creating writer at 1733706242149Flushing 6c6d4fa3aa54648498c3fb38c9152410/testFamily: appending metadata at 1733706242163 (+14 ms)Flushing 6c6d4fa3aa54648498c3fb38c9152410/testFamily: closing flushed file at 1733706242163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6584f386: reopening flushed file at 1733706242631 (+468 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c6d4fa3aa54648498c3fb38c9152410 in 495ms, sequenceid=8, compaction requested=false at 1733706242643 (+12 ms)Writing region close event to WAL at 1733706242643Running coprocessor post-close hooks at 1733706242648 (+5 ms)Closed at 1733706242648 2024-12-09T01:04:02,648 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733706204159.6c6d4fa3aa54648498c3fb38c9152410. 2024-12-09T01:04:02,649 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29ad3b85d9009d8733d9b57c8cdf387f, disabling compactions & flushes 2024-12-09T01:04:02,649 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:04:02,649 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:04:02,649 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. after waiting 0 ms 2024-12-09T01:04:02,649 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:04:02,649 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 29ad3b85d9009d8733d9b57c8cdf387f 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:02,663 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/.tmp/testFamily/1e4cd24c50ee4cf2b062df884867462c is 61, key is row-0/testFamily:testQualifier/1733706213297/Put/seqid=0 2024-12-09T01:04:02,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741863_1039 (size=5049) 2024-12-09T01:04:02,684 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-09T01:04:02,684 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1325): Online Regions={29ad3b85d9009d8733d9b57c8cdf387f=testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f., b36dedecc3c4f56c9617b0a8a6b58ae1=testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., 68c0546b8ae3c545b7fd61a2ea54459d=hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., 5fdfc6cd0e1d087d33da2039f5cfb09e=testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., 635b2e88cb0de9431907c0b036d819ae=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.} 2024-12-09T01:04:02,685 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:02,885 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:03,069 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/.tmp/testFamily/1e4cd24c50ee4cf2b062df884867462c 2024-12-09T01:04:03,083 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/.tmp/testFamily/1e4cd24c50ee4cf2b062df884867462c as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/testFamily/1e4cd24c50ee4cf2b062df884867462c 2024-12-09T01:04:03,085 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 29ad3b85d9009d8733d9b57c8cdf387f, 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:03,092 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/testFamily/1e4cd24c50ee4cf2b062df884867462c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:03,093 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29ad3b85d9009d8733d9b57c8cdf387f in 444ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:03,098 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnNext/29ad3b85d9009d8733d9b57c8cdf387f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:03,098 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29ad3b85d9009d8733d9b57c8cdf387f: Waiting for close lock at 1733706242649Running coprocessor pre-close hooks at 1733706242649Disabling compacts and flushes for region at 1733706242649Disabling writes for close at 1733706242649Obtaining lock to block concurrent updates at 1733706242649Preparing flush snapshotting stores in 29ad3b85d9009d8733d9b57c8cdf387f at 1733706242649Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706242649Flushing stores of testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. at 1733706242650 (+1 ms)Flushing 29ad3b85d9009d8733d9b57c8cdf387f/testFamily: creating writer at 1733706242650Flushing 29ad3b85d9009d8733d9b57c8cdf387f/testFamily: appending metadata at 1733706242663 (+13 ms)Flushing 29ad3b85d9009d8733d9b57c8cdf387f/testFamily: closing flushed file at 1733706242663Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab9e8b2: reopening flushed file at 1733706243082 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 29ad3b85d9009d8733d9b57c8cdf387f in 444ms, sequenceid=8, compaction requested=false at 1733706243093 (+11 ms)Writing region close event to WAL at 1733706243094 (+1 ms)Running coprocessor post-close hooks at 1733706243098 (+4 ms)Closed at 1733706243098 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733706210255.29ad3b85d9009d8733d9b57c8cdf387f. 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b36dedecc3c4f56c9617b0a8a6b58ae1, disabling compactions & flushes 2024-12-09T01:04:03,099 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. after waiting 0 ms 2024-12-09T01:04:03,099 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:04:03,099 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b36dedecc3c4f56c9617b0a8a6b58ae1 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:03,115 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/.tmp/testFamily/79309d8486654b5a9bd8f5d27b98c546 is 61, key is row-0/testFamily:testQualifier/1733706237580/Put/seqid=0 2024-12-09T01:04:03,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741864_1040 (size=5049) 2024-12-09T01:04:03,286 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:03,486 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d, b36dedecc3c4f56c9617b0a8a6b58ae1 2024-12-09T01:04:03,521 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/.tmp/testFamily/79309d8486654b5a9bd8f5d27b98c546 2024-12-09T01:04:03,535 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/.tmp/testFamily/79309d8486654b5a9bd8f5d27b98c546 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily/79309d8486654b5a9bd8f5d27b98c546 2024-12-09T01:04:03,542 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily/79309d8486654b5a9bd8f5d27b98c546, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:03,543 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b36dedecc3c4f56c9617b0a8a6b58ae1 in 444ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:03,548 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/b36dedecc3c4f56c9617b0a8a6b58ae1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:03,549 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b36dedecc3c4f56c9617b0a8a6b58ae1: Waiting for close lock at 1733706243099Running coprocessor pre-close hooks at 1733706243099Disabling compacts and flushes for region at 1733706243099Disabling writes for close at 1733706243099Obtaining lock to block concurrent updates at 1733706243099Preparing flush snapshotting stores in b36dedecc3c4f56c9617b0a8a6b58ae1 at 1733706243099Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706243100 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. at 1733706243100Flushing b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily: creating writer at 1733706243100Flushing b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily: appending metadata at 1733706243115 (+15 ms)Flushing b36dedecc3c4f56c9617b0a8a6b58ae1/testFamily: closing flushed file at 1733706243115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5de78956: reopening flushed file at 1733706243534 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b36dedecc3c4f56c9617b0a8a6b58ae1 in 444ms, sequenceid=8, compaction requested=false at 1733706243543 (+9 ms)Writing region close event to WAL at 1733706243544 (+1 ms)Running coprocessor post-close hooks at 1733706243549 (+5 ms)Closed at 1733706243549 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733706234537.b36dedecc3c4f56c9617b0a8a6b58ae1. 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 68c0546b8ae3c545b7fd61a2ea54459d, disabling compactions & flushes 2024-12-09T01:04:03,549 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. after waiting 0 ms 2024-12-09T01:04:03,549 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:04:03,549 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 68c0546b8ae3c545b7fd61a2ea54459d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:03,566 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/.tmp/testFamily/77a1e17d00c8467a96c17a4af9d3642b is 61, key is row-0/testFamily:testQualifier/1733706188909/Put/seqid=0 2024-12-09T01:04:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741865_1041 (size=5049) 2024-12-09T01:04:03,687 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-09T01:04:03,687 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1325): Online Regions={68c0546b8ae3c545b7fd61a2ea54459d=hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., 5fdfc6cd0e1d087d33da2039f5cfb09e=testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., 635b2e88cb0de9431907c0b036d819ae=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.} 2024-12-09T01:04:03,687 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:04:03,887 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae, 68c0546b8ae3c545b7fd61a2ea54459d 2024-12-09T01:04:03,973 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/.tmp/testFamily/77a1e17d00c8467a96c17a4af9d3642b 2024-12-09T01:04:03,986 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/.tmp/testFamily/77a1e17d00c8467a96c17a4af9d3642b as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/testFamily/77a1e17d00c8467a96c17a4af9d3642b 2024-12-09T01:04:03,993 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/testFamily/77a1e17d00c8467a96c17a4af9d3642b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:03,994 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 68c0546b8ae3c545b7fd61a2ea54459d in 445ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:03,999 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnNextAsync/68c0546b8ae3c545b7fd61a2ea54459d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:03,999 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:04:03,999 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 68c0546b8ae3c545b7fd61a2ea54459d: Waiting for close lock at 1733706243549Running coprocessor pre-close hooks at 1733706243549Disabling compacts and flushes for region at 1733706243549Disabling writes for close at 1733706243549Obtaining lock to block concurrent updates at 1733706243550 (+1 ms)Preparing flush snapshotting stores in 68c0546b8ae3c545b7fd61a2ea54459d at 1733706243550Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706243550Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. at 1733706243550Flushing 68c0546b8ae3c545b7fd61a2ea54459d/testFamily: creating writer at 1733706243551 (+1 ms)Flushing 68c0546b8ae3c545b7fd61a2ea54459d/testFamily: appending metadata at 1733706243566 (+15 ms)Flushing 68c0546b8ae3c545b7fd61a2ea54459d/testFamily: closing flushed file at 1733706243566Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d9071eb: reopening flushed file at 1733706243985 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 68c0546b8ae3c545b7fd61a2ea54459d in 445ms, sequenceid=8, compaction requested=false at 1733706243994 (+9 ms)Writing region close event to WAL at 1733706243995 (+1 ms)Running coprocessor post-close hooks at 1733706243999 (+4 ms)Closed at 1733706243999 2024-12-09T01:04:03,999 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733706185866.68c0546b8ae3c545b7fd61a2ea54459d. 2024-12-09T01:04:04,000 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5fdfc6cd0e1d087d33da2039f5cfb09e, disabling compactions & flushes 2024-12-09T01:04:04,000 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:04:04,000 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:04:04,000 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. after waiting 0 ms 2024-12-09T01:04:04,000 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:04:04,000 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5fdfc6cd0e1d087d33da2039f5cfb09e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:04,014 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/.tmp/testFamily/5c196539c7774a8696d3b27eaab78c3b is 61, key is row-0/testFamily:testQualifier/1733706198001/Put/seqid=0 2024-12-09T01:04:04,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741866_1042 (size=5049) 2024-12-09T01:04:04,088 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:04,288 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 5fdfc6cd0e1d087d33da2039f5cfb09e, 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:04,420 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/.tmp/testFamily/5c196539c7774a8696d3b27eaab78c3b 2024-12-09T01:04:04,433 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/.tmp/testFamily/5c196539c7774a8696d3b27eaab78c3b as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily/5c196539c7774a8696d3b27eaab78c3b 2024-12-09T01:04:04,441 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily/5c196539c7774a8696d3b27eaab78c3b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:04,442 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5fdfc6cd0e1d087d33da2039f5cfb09e in 442ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:04,447 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/default/testRetryOutOfOrderScannerNextException/5fdfc6cd0e1d087d33da2039f5cfb09e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:04,448 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5fdfc6cd0e1d087d33da2039f5cfb09e: Waiting for close lock at 1733706244000Running coprocessor pre-close hooks at 1733706244000Disabling compacts and flushes for region at 1733706244000Disabling writes for close at 1733706244000Obtaining lock to block concurrent updates at 1733706244000Preparing flush snapshotting stores in 5fdfc6cd0e1d087d33da2039f5cfb09e at 1733706244000Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706244000Flushing stores of testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. at 1733706244001 (+1 ms)Flushing 5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily: creating writer at 1733706244001Flushing 5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily: appending metadata at 1733706244014 (+13 ms)Flushing 5fdfc6cd0e1d087d33da2039f5cfb09e/testFamily: closing flushed file at 1733706244014Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d27602b: reopening flushed file at 1733706244431 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5fdfc6cd0e1d087d33da2039f5cfb09e in 442ms, sequenceid=8, compaction requested=false at 1733706244442 (+11 ms)Writing region close event to WAL at 1733706244443 (+1 ms)Running coprocessor post-close hooks at 1733706244448 (+5 ms)Closed at 1733706244448 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733706194957.5fdfc6cd0e1d087d33da2039f5cfb09e. 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 635b2e88cb0de9431907c0b036d819ae, disabling compactions & flushes 2024-12-09T01:04:04,448 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. after waiting 0 ms 2024-12-09T01:04:04,448 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:04:04,448 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 635b2e88cb0de9431907c0b036d819ae 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T01:04:04,462 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/.tmp/testFamily/6dcdcecc05b1431288768ff13d467444 is 61, key is row-0/testFamily:testQualifier/1733706219411/Put/seqid=0 2024-12-09T01:04:04,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741867_1043 (size=5049) 2024-12-09T01:04:04,488 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:04,689 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T01:04:04,689 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1325): Online Regions={635b2e88cb0de9431907c0b036d819ae=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae.} 2024-12-09T01:04:04,689 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:04,869 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/.tmp/testFamily/6dcdcecc05b1431288768ff13d467444 2024-12-09T01:04:04,883 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/.tmp/testFamily/6dcdcecc05b1431288768ff13d467444 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/testFamily/6dcdcecc05b1431288768ff13d467444 2024-12-09T01:04:04,889 DEBUG [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1351): Waiting on 635b2e88cb0de9431907c0b036d819ae 2024-12-09T01:04:04,891 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/testFamily/6dcdcecc05b1431288768ff13d467444, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T01:04:04,892 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 635b2e88cb0de9431907c0b036d819ae in 444ms, sequenceid=8, compaction requested=false 2024-12-09T01:04:04,898 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/635b2e88cb0de9431907c0b036d819ae/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T01:04:04,899 INFO [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:04:04,899 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 635b2e88cb0de9431907c0b036d819ae: Waiting for close lock at 1733706244448Running coprocessor pre-close hooks at 1733706244448Disabling compacts and flushes for region at 1733706244448Disabling writes for close at 1733706244448Obtaining lock to block concurrent updates at 1733706244448Preparing flush snapshotting stores in 635b2e88cb0de9431907c0b036d819ae at 1733706244449 (+1 ms)Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733706244449Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. at 1733706244449Flushing 635b2e88cb0de9431907c0b036d819ae/testFamily: creating writer at 1733706244449Flushing 635b2e88cb0de9431907c0b036d819ae/testFamily: appending metadata at 1733706244462 (+13 ms)Flushing 635b2e88cb0de9431907c0b036d819ae/testFamily: closing flushed file at 1733706244462Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60a502ac: reopening flushed file at 1733706244882 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 635b2e88cb0de9431907c0b036d819ae in 444ms, sequenceid=8, compaction requested=false at 1733706244892 (+10 ms)Writing region close event to WAL at 1733706244893 (+1 ms)Running coprocessor post-close hooks at 1733706244899 (+6 ms)Closed at 1733706244899 2024-12-09T01:04:04,900 DEBUG [RS_CLOSE_REGION-regionserver/1bae0942de96:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733706216351.635b2e88cb0de9431907c0b036d819ae. 2024-12-09T01:04:05,090 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(976): stopping server 1bae0942de96,46191,1733706140132; all regions closed. 2024-12-09T01:04:05,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741834_1010 (size=19479) 2024-12-09T01:04:05,108 DEBUG [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/oldWALs 2024-12-09T01:04:05,108 INFO [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1bae0942de96%2C46191%2C1733706140132.meta:.meta(num 1733706173117) 2024-12-09T01:04:05,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741833_1009 (size=20031) 2024-12-09T01:04:05,115 DEBUG [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/oldWALs 2024-12-09T01:04:05,115 INFO [RS:0;1bae0942de96:46191 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1bae0942de96%2C46191%2C1733706140132:(num 1733706172693) 2024-12-09T01:04:05,115 DEBUG [RS:0;1bae0942de96:46191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:04:05,115 INFO [RS:0;1bae0942de96:46191 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:04:05,115 INFO [RS:0;1bae0942de96:46191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:04:05,116 INFO [RS:0;1bae0942de96:46191 {}] hbase.ChoreService(370): Chore service for: regionserver/1bae0942de96:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T01:04:05,116 INFO [RS:0;1bae0942de96:46191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:04:05,116 INFO [regionserver/1bae0942de96:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:04:05,117 INFO [RS:0;1bae0942de96:46191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46191 2024-12-09T01:04:05,159 INFO [regionserver/1bae0942de96:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:04:05,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:04:05,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1bae0942de96,46191,1733706140132 2024-12-09T01:04:05,176 INFO [RS:0;1bae0942de96:46191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:04:05,185 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1bae0942de96,46191,1733706140132] 2024-12-09T01:04:05,192 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1bae0942de96,46191,1733706140132 already deleted, retry=false 2024-12-09T01:04:05,193 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1bae0942de96,46191,1733706140132 expired; onlineServers=0 2024-12-09T01:04:05,193 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1bae0942de96,34071,1733706138987' ***** 2024-12-09T01:04:05,193 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T01:04:05,193 INFO [M:0;1bae0942de96:34071 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:04:05,194 INFO [M:0;1bae0942de96:34071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:04:05,194 DEBUG [M:0;1bae0942de96:34071 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T01:04:05,194 DEBUG [M:0;1bae0942de96:34071 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T01:04:05,194 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T01:04:05,194 DEBUG [master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.small.0-1733706142493 {}] cleaner.HFileCleaner(306): Exit Thread[master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.small.0-1733706142493,5,FailOnTimeoutGroup] 2024-12-09T01:04:05,194 DEBUG [master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.large.0-1733706142491 {}] cleaner.HFileCleaner(306): Exit Thread[master/1bae0942de96:0:becomeActiveMaster-HFileCleaner.large.0-1733706142491,5,FailOnTimeoutGroup] 2024-12-09T01:04:05,195 INFO [M:0;1bae0942de96:34071 {}] hbase.ChoreService(370): Chore service for: master/1bae0942de96:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T01:04:05,195 INFO [M:0;1bae0942de96:34071 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:04:05,195 DEBUG [M:0;1bae0942de96:34071 {}] master.HMaster(1795): Stopping service threads 2024-12-09T01:04:05,196 INFO [M:0;1bae0942de96:34071 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T01:04:05,196 INFO [M:0;1bae0942de96:34071 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:04:05,197 INFO [M:0;1bae0942de96:34071 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T01:04:05,197 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T01:04:05,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T01:04:05,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:04:05,201 DEBUG [M:0;1bae0942de96:34071 {}] zookeeper.ZKUtil(347): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T01:04:05,201 WARN [M:0;1bae0942de96:34071 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T01:04:05,202 INFO [M:0;1bae0942de96:34071 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/.lastflushedseqids 2024-12-09T01:04:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741868_1044 (size=649) 2024-12-09T01:04:05,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:04:05,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x100082600dc0001, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:04:05,286 INFO [RS:0;1bae0942de96:46191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:04:05,286 INFO [RS:0;1bae0942de96:46191 {}] regionserver.HRegionServer(1031): Exiting; stopping=1bae0942de96,46191,1733706140132; zookeeper connection closed. 2024-12-09T01:04:05,287 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4795195d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4795195d 2024-12-09T01:04:05,288 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T01:04:05,617 INFO [M:0;1bae0942de96:34071 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T01:04:05,617 INFO [M:0;1bae0942de96:34071 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T01:04:05,618 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:04:05,618 INFO [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:04:05,618 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:04:05,618 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:04:05,618 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:04:05,619 INFO [M:0;1bae0942de96:34071 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.97 KB heapSize=190.84 KB 2024-12-09T01:04:05,640 DEBUG [M:0;1bae0942de96:34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57a2aaa5a15c4a678cf07f04d7e5e606 is 82, key is hbase:meta,,1/info:regioninfo/1733706173196/Put/seqid=0 2024-12-09T01:04:05,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741869_1045 (size=5672) 2024-12-09T01:04:06,047 INFO [M:0;1bae0942de96:34071 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57a2aaa5a15c4a678cf07f04d7e5e606 2024-12-09T01:04:06,074 DEBUG [M:0;1bae0942de96:34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f934e7b2aec46e19c210033376434ee is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733706174853/Put/seqid=0 2024-12-09T01:04:06,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741870_1046 (size=14068) 2024-12-09T01:04:06,307 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:04:06,481 INFO [M:0;1bae0942de96:34071 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.41 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f934e7b2aec46e19c210033376434ee 2024-12-09T01:04:06,493 INFO [M:0;1bae0942de96:34071 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f934e7b2aec46e19c210033376434ee 2024-12-09T01:04:06,510 DEBUG [M:0;1bae0942de96:34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d63657fa7ebb4681a934c3566d0dc715 is 69, key is 1bae0942de96,46191,1733706140132/rs:state/1733706142539/Put/seqid=0 2024-12-09T01:04:06,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741871_1047 (size=5156) 2024-12-09T01:04:06,917 INFO [M:0;1bae0942de96:34071 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d63657fa7ebb4681a934c3566d0dc715 2024-12-09T01:04:06,931 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/57a2aaa5a15c4a678cf07f04d7e5e606 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/57a2aaa5a15c4a678cf07f04d7e5e606 2024-12-09T01:04:06,938 INFO [M:0;1bae0942de96:34071 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/57a2aaa5a15c4a678cf07f04d7e5e606, entries=8, sequenceid=317, filesize=5.5 K 2024-12-09T01:04:06,939 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0f934e7b2aec46e19c210033376434ee as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f934e7b2aec46e19c210033376434ee 2024-12-09T01:04:06,946 INFO [M:0;1bae0942de96:34071 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f934e7b2aec46e19c210033376434ee 2024-12-09T01:04:06,946 INFO [M:0;1bae0942de96:34071 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0f934e7b2aec46e19c210033376434ee, entries=33, sequenceid=317, filesize=13.7 K 2024-12-09T01:04:06,947 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d63657fa7ebb4681a934c3566d0dc715 as hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d63657fa7ebb4681a934c3566d0dc715 2024-12-09T01:04:06,953 INFO [M:0;1bae0942de96:34071 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41839/user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d63657fa7ebb4681a934c3566d0dc715, entries=1, sequenceid=317, filesize=5.0 K 2024-12-09T01:04:06,954 INFO [M:0;1bae0942de96:34071 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.54 KB/195112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1336ms, sequenceid=317, compaction requested=false 2024-12-09T01:04:06,956 INFO [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:04:06,956 DEBUG [M:0;1bae0942de96:34071 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733706245618Disabling compacts and flushes for region at 1733706245618Disabling writes for close at 1733706245618Obtaining lock to block concurrent updates at 1733706245619 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733706245619Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166879, getHeapSize=195352, getOffHeapSize=0, getCellsCount=364 at 1733706245619Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733706245621 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733706245621Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733706245639 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733706245639Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733706246059 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733706246074 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733706246074Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733706246493 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733706246510 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733706246510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20192fd4: reopening flushed file at 1733706246930 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d157127: reopening flushed file at 1733706246938 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72723215: reopening flushed file at 1733706246946 (+8 ms)Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.54 KB/195112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1336ms, sequenceid=317, compaction requested=false at 1733706246954 (+8 ms)Writing region close event to WAL at 1733706246956 (+2 ms)Closed at 1733706246956 2024-12-09T01:04:06,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44185 is added to blk_1073741830_1006 (size=188091) 2024-12-09T01:04:06,959 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3a094926-93fc-f2be-18c2-16348ed459f0/MasterData/WALs/1bae0942de96,34071,1733706138987/1bae0942de96%2C34071%2C1733706138987.1733706141861 not finished, retry = 0 2024-12-09T01:04:07,060 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:04:07,061 INFO [M:0;1bae0942de96:34071 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T01:04:07,061 INFO [M:0;1bae0942de96:34071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34071 2024-12-09T01:04:07,061 INFO [M:0;1bae0942de96:34071 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:04:07,210 INFO [M:0;1bae0942de96:34071 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:04:07,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:04:07,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34071-0x100082600dc0000, quorum=127.0.0.1:57232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:04:07,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc78d53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T01:04:07,269 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2120cdf2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T01:04:07,269 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T01:04:07,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9495fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T01:04:07,270 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@537310ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.log.dir/,STOPPED} 2024-12-09T01:04:07,271 WARN [BP-1050418544-172.17.0.2-1733706135545 heartbeating to localhost/127.0.0.1:41839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T01:04:07,271 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T01:04:07,271 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T01:04:07,271 WARN [BP-1050418544-172.17.0.2-1733706135545 heartbeating to localhost/127.0.0.1:41839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1050418544-172.17.0.2-1733706135545 (Datanode Uuid 096d782a-4d07-48af-9b90-b523295ed21d) service to localhost/127.0.0.1:41839 2024-12-09T01:04:07,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/data/data1/current/BP-1050418544-172.17.0.2-1733706135545 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T01:04:07,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/cluster_9db1c3f5-63e6-d06f-34de-821fd4c62c8f/data/data2/current/BP-1050418544-172.17.0.2-1733706135545 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T01:04:07,273 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T01:04:07,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@562f0d0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T01:04:07,281 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@281b58fb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T01:04:07,281 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T01:04:07,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ac7ce20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T01:04:07,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33bab939{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bfa22bc6-456b-e42b-cd43-91e845ac11a5/hadoop.log.dir/,STOPPED} 2024-12-09T01:04:07,287 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T01:04:07,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down